Mostrar articulos que matchean contenido

.gitignore

@@ -0,0 +1 @@
+test.db

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "go-rss"]
+	path = go-rss
+	url = https://gitea.nulo.in/Nulo/go-rss

fetcher.go

@@ -0,0 +1,96 @@
+package main
+
+import (
+	"database/sql"
+	"log"
+	"time"
+
+	rss "github.com/ungerik/go-rss"
+)
+
+type GenericRSSFetcher struct {
+	// MedioName es el nombre para poner en la BD (field "medio")
+	MedioName string
+	FeedURL   string
+	DB        *sql.DB
+
+	// SubsetName es un nombre para un feed específico que se usa para debug
+	SubsetName string
+
+	PubDateFormat string
+}
+
+// SQLiteTimestampFormats[0]
+const DefaultDateFormat = "2006-01-02 15:04:05.999999999-07:00"
+
+// TODO: Capturar errores
+func (fetcher GenericRSSFetcher) Fetch() {
+	resp, err := rss.Read(fetcher.FeedURL, false)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	feed, err := rss.Regular(resp)
+	if err != nil {
+		log.Panicln(err)
+	}
+
+	for _, entry := range feed.Item {
+
+		var parsedDate time.Time
+
+		if len(fetcher.PubDateFormat) != 0 {
+			parsedDate, err = entry.PubDate.ParseWithFormat(fetcher.PubDateFormat)
+		} else {
+			parsedDate, err = entry.PubDate.Parse()
+		}
+		if err != nil {
+			log.Panicln(err)
+		}
+
+		formatted_pub_date := parsedDate.Format(DefaultDateFormat)
+
+		_, err = fetcher.DB.Exec(`
+		INSERT INTO notas(medio, title, content, link, publication_date)
+		VALUES(?, ?, ?, ?, ?)
+		ON CONFLICT(link) DO UPDATE SET
+			title=excluded.title,
+			content=excluded.content;
+	        `, fetcher.MedioName, entry.Title, entry.ContentEncoded, entry.Link, formatted_pub_date)
+		if err != nil {
+			log.Println("Error when saving nota", err)
+		}
+	}
+	log.Printf("[%s/%s] Procesé %d notas", fetcher.MedioName, fetcher.SubsetName, len(feed.Item))
+}
+
+func Pagina12(db *sql.DB, subsetName string, url string) GenericRSSFetcher {
+	return GenericRSSFetcher{
+		MedioName:     "pagina_12_rss",
+		SubsetName:    subsetName,
+		FeedURL:       url,
+		DB:            db,
+		PubDateFormat: time.RFC1123,
+	}
+}
+
+func cronologicalFetcher(db *sql.DB) {
+	for true {
+		GenericRSSFetcher{
+			MedioName: "la_nacion_rss",
+			FeedURL:   "https://www.lanacion.com.ar/arcio/rss/",
+			DB:        db,
+		}.Fetch()
+		GenericRSSFetcher{
+			MedioName: "infobae",
+			FeedURL:   "https://www.infobae.com/feeds/rss/",
+			DB:        db,
+		}.Fetch()
+		Pagina12(db, "portada", "https://www.pagina12.com.ar/rss/portada").Fetch()
+		Pagina12(db, "edicion-impresa", "https://www.pagina12.com.ar/rss/edicion-impresa").Fetch()
+		Pagina12(db, "secciones/el-pais", "https://www.pagina12.com.ar/rss/secciones/el-pais/notas").Fetch()
+		Pagina12(db, "secciones/sociedad", "https://www.pagina12.com.ar/rss/secciones/sociedad/notas").Fetch()
+		time.Sleep(time.Minute * 2)
+	}
+
+}

go-rss

@@ -0,0 +1 @@
+Subproject commit 1b4d16383269a0a2b5e51aae3d166a7cb1978a9f

go.mod

@@ -0,0 +1,12 @@
+module medios.nulo.in/web
+
+go 1.19
+
+require (
+	github.com/mattn/go-sqlite3 v1.14.15
+	github.com/ungerik/go-rss v0.0.0-20200405130036-81ac15598626
+)
+
+require github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c // indirect
+
+replace github.com/ungerik/go-rss => ./go-rss

go.sum

@@ -0,0 +1,10 @@
+gitea.nulo.in/Nulo/go-rss v0.0.0-20200405130036-81ac15598626 h1:QkC7ZtrV7w6LON50hPGuXOz6J4196WIuCC+jRnYFlko=
+gitea.nulo.in/Nulo/go-rss v0.0.0-20200405130036-81ac15598626/go.mod h1:UeGmbG1AM6Cx0FfruZOzOC02lDvXbUWV+06tAxgCoGA=
+gitea.nulo.in/Nulo/go-rss v0.0.0-20221012153719-f25f0ae1f12e h1:kd56QKzyFz2ALRN5ITosKINKKo+hPLn4EW2agt2eUOM=
+gitea.nulo.in/Nulo/go-rss v0.0.0-20221012153719-f25f0ae1f12e/go.mod h1:UeGmbG1AM6Cx0FfruZOzOC02lDvXbUWV+06tAxgCoGA=
+github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
+github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c h1:P6XGcuPTigoHf4TSu+3D/7QOQ1MbL6alNwrGhcW7sKw=
+github.com/paulrosania/go-charset v0.0.0-20190326053356-55c9d7a5834c/go.mod h1:YnNlZP7l4MhyGQ4CBRwv6ohZTPrUJJZtEv4ZgADkbs4=
+github.com/ungerik/go-rss v0.0.0-20200405130036-81ac15598626 h1:kCZvptlArqbfrFWL0V0YWwV9iXEvMKaUMjJvXQ/wsE4=
+github.com/ungerik/go-rss v0.0.0-20200405130036-81ac15598626/go.mod h1:UeGmbG1AM6Cx0FfruZOzOC02lDvXbUWV+06tAxgCoGA=

main.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"html"
+	"log"
+	"net/http"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+// type Nota struct {
+// 	Medio           string `json:"medio"`
+// 	Title           string `json:"title"`
+// 	Link            string `json:"link"`
+// 	PublicationDate string `json:"publication_date" db:"publication_date"`
+// 	CreatedAt       string `json:"created_at" db:"created_at"`
+// }
+
+type fooHandler struct {
+	DB *sql.DB
+}
+
+func (foo fooHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+
+	fmt.Fprintf(w, `<!doctype html>
+	<meta charset="utf-8">
+	<title>Observatorio de medios</title>
+	<ul>
+	`)
+
+	rows, err := foo.DB.Query(`
+	select medio, title, link, publication_date
+	from notas
+	where
+		title like "%colegio%" or
+		title like "%escuela%" or
+		title like "%Acosta%" or
+		title like "%tomas%" or
+		content like "%colegio%" or
+		content like "%escuela%" or
+		content like "%Acosta%" or
+		content like "%tomas%"
+	order by publication_date asc
+	limit 50
+	`)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var medio string
+		var title string
+		var link string
+		var publication_date string
+		err = rows.Scan(&medio, &title, &link, &publication_date)
+		if err != nil {
+			log.Fatal(err)
+		}
+		fmt.Fprintf(w, "<li>[%s] <a href=\"%s\">%s</a> %s\n",
+			html.EscapeString(medio),
+			html.EscapeString(link),
+			html.EscapeString(title),
+			html.EscapeString(publication_date),
+		)
+	}
+	err = rows.Err()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	fmt.Fprintf(w, `</ul>`)
+}
+
+func main() {
+	db, err := sql.Open("sqlite3", "./test.db?cache=shared")
+	if err != nil {
+		log.Panic(err)
+	}
+	defer db.Close()
+
+	db.SetMaxOpenConns(1)
+
+	http.Handle("/foo", fooHandler{db})
+
+	go cronologicalFetcher(db)
+
+	log.Println("Listening in :8080")
+	log.Fatal(http.ListenAndServe(":8080", nil))
+}

migrations/00001-articulos.sql

@@ -0,0 +1,9 @@
+CREATE TABLE notas (
+	medio TEXT NOT NULL,
+	title TEXT NOT NULL,
+	content TEXT,
+	link TEXT NOT NULL,
+	publication_date DATETIME NOT NULL,
+	created_at DATETIME NOT NULL DEFAULT current_timestamp,
+	UNIQUE(link)
+);