Compare commits
No commits in common. "e5a6a1a073eb8d9f11696f253d6dd82fd433d012" and "49415ec22669d922e04871ce52bf96b99a9d1a87" have entirely different histories.
e5a6a1a073
...
49415ec226
8 changed files with 0 additions and 224 deletions
1
.gitignore
vendored
1
.gitignore
vendored
|
|
@ -1 +0,0 @@
|
|||
test.db
|
||||
3
.gitmodules
vendored
3
.gitmodules
vendored
|
|
@ -1,3 +0,0 @@
|
|||
[submodule "go-rss"]
|
||||
path = go-rss
|
||||
url = https://gitea.nulo.in/Nulo/go-rss
|
||||
96
fetcher.go
96
fetcher.go
|
|
@ -1,96 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
rss "github.com/ungerik/go-rss"
|
||||
)
|
||||
|
||||
type GenericRSSFetcher struct {
|
||||
// MedioName es el nombre para poner en la BD (field "medio")
|
||||
MedioName string
|
||||
FeedURL string
|
||||
DB *sql.DB
|
||||
|
||||
// SubsetName es un nombre para un feed específico que se usa para debug
|
||||
SubsetName string
|
||||
|
||||
PubDateFormat string
|
||||
}
|
||||
|
||||
// SQLiteTimestampFormats[0]
|
||||
const DefaultDateFormat = "2006-01-02 15:04:05.999999999-07:00"
|
||||
|
||||
// TODO: Capturar errores
|
||||
func (fetcher GenericRSSFetcher) Fetch() {
|
||||
resp, err := rss.Read(fetcher.FeedURL, false)
|
||||
if err != nil {
|
||||
log.Panicln(err)
|
||||
}
|
||||
|
||||
feed, err := rss.Regular(resp)
|
||||
if err != nil {
|
||||
log.Panicln(err)
|
||||
}
|
||||
|
||||
for _, entry := range feed.Item {
|
||||
|
||||
var parsedDate time.Time
|
||||
|
||||
if len(fetcher.PubDateFormat) != 0 {
|
||||
parsedDate, err = entry.PubDate.ParseWithFormat(fetcher.PubDateFormat)
|
||||
} else {
|
||||
parsedDate, err = entry.PubDate.Parse()
|
||||
}
|
||||
if err != nil {
|
||||
log.Panicln(err)
|
||||
}
|
||||
|
||||
formatted_pub_date := parsedDate.Format(DefaultDateFormat)
|
||||
|
||||
_, err = fetcher.DB.Exec(`
|
||||
INSERT INTO notas(medio, title, content, link, publication_date)
|
||||
VALUES(?, ?, ?, ?, ?)
|
||||
ON CONFLICT(link) DO UPDATE SET
|
||||
title=excluded.title,
|
||||
content=excluded.content;
|
||||
`, fetcher.MedioName, entry.Title, entry.ContentEncoded, entry.Link, formatted_pub_date)
|
||||
if err != nil {
|
||||
log.Println("Error when saving nota", err)
|
||||
}
|
||||
}
|
||||
log.Printf("[%s/%s] Procesé %d notas", fetcher.MedioName, fetcher.SubsetName, len(feed.Item))
|
||||
}
|
||||
|
||||
func Pagina12(db *sql.DB, subsetName string, url string) GenericRSSFetcher {
|
||||
return GenericRSSFetcher{
|
||||
MedioName: "pagina_12_rss",
|
||||
SubsetName: subsetName,
|
||||
FeedURL: url,
|
||||
DB: db,
|
||||
PubDateFormat: time.RFC1123,
|
||||
}
|
||||
}
|
||||
|
||||
func cronologicalFetcher(db *sql.DB) {
|
||||
for true {
|
||||
GenericRSSFetcher{
|
||||
MedioName: "la_nacion_rss",
|
||||
FeedURL: "https://www.lanacion.com.ar/arcio/rss/",
|
||||
DB: db,
|
||||
}.Fetch()
|
||||
GenericRSSFetcher{
|
||||
MedioName: "infobae",
|
||||
FeedURL: "https://www.infobae.com/feeds/rss/",
|
||||
DB: db,
|
||||
}.Fetch()
|
||||
Pagina12(db, "portada", "https://www.pagina12.com.ar/rss/portada").Fetch()
|
||||
Pagina12(db, "edicion-impresa", "https://www.pagina12.com.ar/rss/edicion-impresa").Fetch()
|
||||
Pagina12(db, "secciones/el-pais", "https://www.pagina12.com.ar/rss/secciones/el-pais/notas").Fetch()
|
||||
Pagina12(db, "secciones/sociedad", "https://www.pagina12.com.ar/rss/secciones/sociedad/notas").Fetch()
|
||||
time.Sleep(time.Minute * 2)
|
||||
}
|
||||
|
||||
}
|
||||
1
go-rss
1
go-rss
|
|
@ -1 +0,0 @@
|
|||
Subproject commit 1b4d16383269a0a2b5e51aae3d166a7cb1978a9f
|
||||
12
go.mod
12
go.mod
|
|
@ -1,12 +0,0 @@
|
|||
module medios.nulo.in/web
|
||||
|
||||
go 1.19
|
||||
|
||||
require (
|
||||
github.com/mattn/go-sqlite3 v1.14.15
|
||||
github.com/ungerik/go-rss v0.0.0-20200405130036-81ac15598626
|
||||
)
|
||||
|
||||
require github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c // indirect
|
||||
|
||||
replace github.com/ungerik/go-rss => ./go-rss
|
||||
10
go.sum
10
go.sum
|
|
@ -1,10 +0,0 @@
|
|||
gitea.nulo.in/Nulo/go-rss v0.0.0-20200405130036-81ac15598626 h1:QkC7ZtrV7w6LON50hPGuXOz6J4196WIuCC+jRnYFlko=
|
||||
gitea.nulo.in/Nulo/go-rss v0.0.0-20200405130036-81ac15598626/go.mod h1:UeGmbG1AM6Cx0FfruZOzOC02lDvXbUWV+06tAxgCoGA=
|
||||
gitea.nulo.in/Nulo/go-rss v0.0.0-20221012153719-f25f0ae1f12e h1:kd56QKzyFz2ALRN5ITosKINKKo+hPLn4EW2agt2eUOM=
|
||||
gitea.nulo.in/Nulo/go-rss v0.0.0-20221012153719-f25f0ae1f12e/go.mod h1:UeGmbG1AM6Cx0FfruZOzOC02lDvXbUWV+06tAxgCoGA=
|
||||
github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
|
||||
github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
|
||||
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c h1:P6XGcuPTigoHf4TSu+3D/7QOQ1MbL6alNwrGhcW7sKw=
|
||||
github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c/go.mod h1:YnNlZP7l4MhyGQ4CBRwv6ohZTPrUJJZtEv4ZgADkbs4=
|
||||
github.com/ungerik/go-rss v0.0.0-20200405130036-81ac15598626 h1:kCZvptlArqbfrFWL0V0YWwV9iXEvMKaUMjJvXQ/wsE4=
|
||||
github.com/ungerik/go-rss v0.0.0-20200405130036-81ac15598626/go.mod h1:UeGmbG1AM6Cx0FfruZOzOC02lDvXbUWV+06tAxgCoGA=
|
||||
92
main.go
92
main.go
|
|
@ -1,92 +0,0 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"html"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
// type Nota struct {
|
||||
// Medio string `json:"medio"`
|
||||
// Title string `json:"title"`
|
||||
// Link string `json:"link"`
|
||||
// PublicationDate string `json:"publication_date" db:"publication_date"`
|
||||
// CreatedAt string `json:"created_at" db:"created_at"`
|
||||
// }
|
||||
|
||||
type fooHandler struct {
|
||||
DB *sql.DB
|
||||
}
|
||||
|
||||
func (foo fooHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
fmt.Fprintf(w, `<!doctype html>
|
||||
<meta charset="utf-8">
|
||||
<title>Observatorio de medios</title>
|
||||
<ul>
|
||||
`)
|
||||
|
||||
rows, err := foo.DB.Query(`
|
||||
select medio, title, link, publication_date
|
||||
from notas
|
||||
where
|
||||
title like "%colegio%" or
|
||||
title like "%escuela%" or
|
||||
title like "%Acosta%" or
|
||||
title like "%tomas%" or
|
||||
content like "%colegio%" or
|
||||
content like "%escuela%" or
|
||||
content like "%Acosta%" or
|
||||
content like "%tomas%"
|
||||
order by publication_date asc
|
||||
limit 50
|
||||
`)
|
||||
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var medio string
|
||||
var title string
|
||||
var link string
|
||||
var publication_date string
|
||||
err = rows.Scan(&medio, &title, &link, &publication_date)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
fmt.Fprintf(w, "<li>[%s] <a href=\"%s\">%s</a> %s\n",
|
||||
html.EscapeString(medio),
|
||||
html.EscapeString(link),
|
||||
html.EscapeString(title),
|
||||
html.EscapeString(publication_date),
|
||||
)
|
||||
}
|
||||
err = rows.Err()
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Fprintf(w, `</ul>`)
|
||||
}
|
||||
|
||||
func main() {
|
||||
db, err := sql.Open("sqlite3", "./test.db?cache=shared")
|
||||
if err != nil {
|
||||
log.Panic(err)
|
||||
}
|
||||
defer db.Close()
|
||||
|
||||
db.SetMaxOpenConns(1)
|
||||
|
||||
http.Handle("/foo", fooHandler{db})
|
||||
|
||||
go cronologicalFetcher(db)
|
||||
|
||||
log.Println("Listening in :8080")
|
||||
log.Fatal(http.ListenAndServe(":8080", nil))
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
CREATE TABLE notas (
|
||||
medio TEXT NOT NULL,
|
||||
title TEXT NOT NULL,
|
||||
content TEXT,
|
||||
link TEXT NOT NULL,
|
||||
publication_date DATETIME NOT NULL,
|
||||
created_at DATETIME NOT NULL DEFAULT current_timestamp,
|
||||
UNIQUE(link)
|
||||
);
|
||||
Loading…
Reference in a new issue