diff --git a/.gitignore b/.gitignore index d3beee5..6fc06e8 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,5 @@ _testmain.go *.test *.prof +latest* +report.html diff --git a/IACR-eprint/feedinfo b/IACR-eprint/feedinfo new file mode 100644 index 0000000..d3e9531 --- /dev/null +++ b/IACR-eprint/feedinfo @@ -0,0 +1 @@ +https://eprint.iacr.org/rss/rss.xml 1440 \ No newline at end of file diff --git a/amnesty/feedinfo b/amnesty/feedinfo new file mode 100644 index 0000000..5b691a1 --- /dev/null +++ b/amnesty/feedinfo @@ -0,0 +1 @@ +https://www.amnesty.ca/rss.xml 1440 \ No newline at end of file diff --git a/arxiv-cs.CR/feedinfo b/arxiv-cs.CR/feedinfo new file mode 100644 index 0000000..3f1ba95 --- /dev/null +++ b/arxiv-cs.CR/feedinfo @@ -0,0 +1 @@ +http://export.arxiv.org/rss/cs.CR 1440 \ No newline at end of file diff --git a/bitcoin-releases/feedinfo b/bitcoin-releases/feedinfo new file mode 100644 index 0000000..2b64d7d --- /dev/null +++ b/bitcoin-releases/feedinfo @@ -0,0 +1 @@ +https://bitcoin.org/en/rss/releases.rss 1440 \ No newline at end of file diff --git a/grin-annoucements/feedinfo b/grin-annoucements/feedinfo new file mode 100644 index 0000000..4fa98f3 --- /dev/null +++ b/grin-annoucements/feedinfo @@ -0,0 +1 @@ +https://www.grin-forum.org/c/announce.rss 1440 \ No newline at end of file diff --git a/i2pblog/feedinfo b/i2pblog/feedinfo new file mode 100644 index 0000000..6653067 --- /dev/null +++ b/i2pblog/feedinfo @@ -0,0 +1 @@ +https://geti2p.net/el/feed/blog/atom 1440 \ No newline at end of file diff --git a/journalist-leighcuen/feedinfo b/journalist-leighcuen/feedinfo new file mode 100644 index 0000000..a0ffea4 --- /dev/null +++ b/journalist-leighcuen/feedinfo @@ -0,0 +1 @@ +https://coindesk.com/author/lcuen/feed 60 \ No newline at end of file diff --git a/journalist-zackwhittaker/feedinfo b/journalist-zackwhittaker/feedinfo new file mode 100644 index 0000000..1c24eff --- /dev/null +++ b/journalist-zackwhittaker/feedinfo @@ -0,0 +1 @@ +https://techcrunch.com/author/zack-whittaker/feed/ 60 \ No newline at end of file diff --git a/katzenpost-blog/feedinfo b/katzenpost-blog/feedinfo new file mode 100644 index 0000000..ed813db --- /dev/null +++ b/katzenpost-blog/feedinfo @@ -0,0 +1 @@ +https://katzenpost.mixnetworks.org/blog/atom.xml 1440 \ No newline at end of file diff --git a/main.go b/main.go new file mode 100644 index 0000000..8374927 --- /dev/null +++ b/main.go @@ -0,0 +1,233 @@ +package main + +import ( + "bufio" + "encoding/json" + "fmt" + "github.com/grokify/html-strip-tags-go" + "github.com/mmcdole/gofeed" + "github.com/writeas/go-strip-markdown" + "golang.org/x/net/proxy" + "io/ioutil" + "log" + "net/http" + "net/url" + "os" + "path" + "strconv" + "strings" + "time" +) + +func fetch(url string, cachepath string) { + fmt.Printf("Fetching [%v]\n", url) + fp := gofeed.NewParser() + feed, err := fp.ParseURL(url) + fmt.Printf("Feed %v %v\n", feed, err) + cache, _ := json.Marshal(feed) + ioutil.WriteFile(cachepath, cache, 0644) +} + +func report() map[string]gofeed.Feed { + feedMap := make(map[string]gofeed.Feed) + items, _ := ioutil.ReadDir(".") + for _, item := range items { + if item.IsDir() { + //subitems, _ := ioutil.ReadDir(item.Name()) + //for _, subitem := range subitems { + cachepath := path.Join(".", item.Name(), "latest") + var feed gofeed.Feed + data, err := ioutil.ReadFile(cachepath) + if err == nil { + json.Unmarshal(data, &feed) + // fmt.Printf("Feed %v\n", feed) + } + feedMap[item.Name()] = feed + //} + } + } + return feedMap +} + +func download(url string, cachepath string) { + fmt.Printf("Fetching [%v]\n", url) + torDialer, err := proxy.SOCKS5("tcp", "127.0.0.1:9050", nil, proxy.Direct) + transportConfig := &http.Transport{ + Dial: torDialer.Dial, + } + client := http.Client{ + Transport: transportConfig, + CheckRedirect: func(r *http.Request, via []*http.Request) error { + r.URL.Opaque = r.URL.Path + return nil + }, + } + + resp, err := client.Get(url) + if err == nil { + defer resp.Body.Close() + body, _ := ioutil.ReadAll(resp.Body) + ioutil.WriteFile(cachepath, body, 0644) + } +} + +func update() { + items, _ := ioutil.ReadDir(".") + for _, item := range items { + if item.IsDir() { + subitems, _ := ioutil.ReadDir(item.Name()) + for _, subitem := range subitems { + if subitem.Name() == "feedinfo" { + // handle file there + filepath := path.Join(".", item.Name(), subitem.Name()) + fmt.Println("Processing: " + filepath) + rawfeedinfo, _ := ioutil.ReadFile(filepath) + feedinfo := strings.Split(strings.TrimSpace(string(rawfeedinfo)), " ") + cachepath := path.Join(".", item.Name(), "latest") + info, err := os.Stat(cachepath) + if err == nil { + duration := time.Since(info.ModTime()) + cron, _ := strconv.Atoi(feedinfo[1]) + // If it has been greater than minutes since the last fetch, we fetch + if (time.Duration(cron) * time.Minute).Minutes() < duration.Minutes() { + fetch(feedinfo[0], cachepath) + } else { + + } + } else { + // First time + fetch(feedinfo[0], cachepath) + } + } else if subitem.Name() == "images" { + filepath := path.Join(".", item.Name(), subitem.Name()) + file, err := os.Open(filepath) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + image := scanner.Text() + imageinfo := strings.Split(image, " ") + uri, _ := url.Parse(imageinfo[0]) + cachepath := path.Join(".", item.Name(), "latest-"+path.Base(uri.Path)) + info, err := os.Stat(cachepath) + if err == nil { + duration := time.Since(info.ModTime()) + cron, _ := strconv.Atoi(imageinfo[1]) + // If it has been greater than minutes since the last fetch, we fetch + if (time.Duration(cron) * time.Minute).Minutes() < duration.Minutes() { + download(imageinfo[0], cachepath) + } else { + + } + } else { + // First time + download(imageinfo[0], cachepath) + } + } + } + } + } + } +} + +func processItems(format []string, items []*gofeed.Item, dateCheck time.Duration) { + num := 0 + for _, item := range items { + if dateCheck == 0 { + processItem(format, *item) + num = 100 + } else { + if item.PublishedParsed != nil { + if time.Since(*item.PublishedParsed) < dateCheck { + processItem(format, *item) + num++ + } + } else if item.UpdatedParsed != nil { + if time.Since(*item.UpdatedParsed) < dateCheck { + processItem(format, *item) + num++ + } + } + } + } + if num == 0 { + fmt.Printf("* Nothing new this %v From %v\n", format[1], format[0]) + } +} + +// stripString removes html, then potential markdown characters, and then some additional potential markdown +func stripString(input string) string { + return strings.Replace(stripmd.Strip(strip.StripTags(input)), "`", "", -1) +} + +func processItem(format []string, item gofeed.Item) { + fmt.Printf("* ") + for i := 2; i < len(format); i++ { + switch format[i] { + case "Title": + fmt.Printf("%v ", stripString(item.Title)) + case "Link": + fmt.Printf("[%v](%v)", stripString(item.Link), stripString(item.Link)) + case "Description": + fmt.Printf("\n * %v
", strings.Replace(stripString(item.Description), "\n", "", -1)) + } + } + fmt.Printf("\n") +} + +func main() { + if len(os.Args) >= 2 { + cmd := os.Args[1] + switch cmd { + case "update": + update() + case "report": + if len(os.Args) == 3 { + feeds := report() + file, err := os.Open(os.Args[2]) + if err != nil { + log.Fatal(err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + reportLine := strings.Split(line, " ") + if strings.HasPrefix(reportLine[0], "%") { + // Ignore, this is a comment + } else if strings.HasPrefix(reportLine[0], "#") || strings.HasPrefix(reportLine[0], "<") { + fmt.Printf("%v\n", strings.TrimSpace(line)) + } else if len(reportLine) > 2 { + _, exists := feeds[reportLine[0]] + if exists { + if reportLine[1] == "ALL" { + processItems(reportLine, feeds[reportLine[0]].Items, 0) + } else if reportLine[1] == "DAY" { // Only output entries from the last Day + processItems(reportLine, feeds[reportLine[0]].Items, time.Hour*24) + } else if reportLine[1] == "WEEK" { // Only output entries from the last Week + processItems(reportLine, feeds[reportLine[0]].Items, time.Hour*24*7) + } else { + index, _ := strconv.Atoi(reportLine[1]) + processItem(reportLine, *feeds[reportLine[0]].Items[index]) + } + } else { + log.Fatalf("Report Template Contains Non-Existent Feed %v\n", reportLine[0]) + } + } else { + fmt.Printf("\n") + } + } + + if err := scanner.Err(); err != nil { + log.Fatal(err) + } + } + } + + } + os.Exit(0) +} diff --git a/monero/feedinfo b/monero/feedinfo new file mode 100644 index 0000000..e45038d --- /dev/null +++ b/monero/feedinfo @@ -0,0 +1 @@ +https://web.getmonero.org/feed.xml 1440 \ No newline at end of file diff --git a/privacy-international/feedinfo b/privacy-international/feedinfo new file mode 100644 index 0000000..6bf6cfd --- /dev/null +++ b/privacy-international/feedinfo @@ -0,0 +1 @@ +https://privacyinternational.org/rss.xml 1440 \ No newline at end of file diff --git a/proof-of-work/feedinfo b/proof-of-work/feedinfo new file mode 100644 index 0000000..6928447 --- /dev/null +++ b/proof-of-work/feedinfo @@ -0,0 +1 @@ +https://proofofwork.news/feed/ 1440 \ No newline at end of file diff --git a/pt-reyes/images b/pt-reyes/images new file mode 100644 index 0000000..9b0f449 --- /dev/null +++ b/pt-reyes/images @@ -0,0 +1,2 @@ +https://ocean.weather.gov/shtml/pyba01bw.gif 120 +https://ocean.weather.gov/shtml/EVPN13.jpg 120 \ No newline at end of file diff --git a/reddit-rtlsdr/feedinfo b/reddit-rtlsdr/feedinfo new file mode 100644 index 0000000..60577b0 --- /dev/null +++ b/reddit-rtlsdr/feedinfo @@ -0,0 +1 @@ +https://www.reddit.com/r/RTLSDR.xml 1440 \ No newline at end of file diff --git a/report.template b/report.template new file mode 100644 index 0000000..6d048df --- /dev/null +++ b/report.template @@ -0,0 +1,75 @@ + + +# Daily Report + +## Weather + +weather 1 Title +weather 2 Title + +## Satellite + + + + +
+ +## In The News + +% Not all the journalists I follow have dedicated rss feeds available for their publication (and some don't have a publication) +journalist-zackwhittaker DAY Title Link +journalist-leighcuen DAY Title Link +reveal DAY Title Link + +## Activism + +seashepherd WEEK Title Link +amnesty WEEK Title Link +privacy-international WEEK Title Link + +## Anonymity Blogs + +tor-blog DAY Title Link +i2pblog DAY Title Link +katzenpost-blog DAY Title Link +securedrop WEEK Title Link + +
+ +## Cryptocurrency + +### General + +proof-of-work WEEK Title Link + +### Bitcoin + +bitcoin-releases 0 Title Link + +### Monero + +monero WEEK Title Link + +### Zcash +zcash-blog WEEK Title Link +zcash-fnd-blog WEEK Title Link + +### Grin +grin-annoucements WEEK Title Link + +
+ +## New Crypto/Security Papers + +% Neither of these offer proper published dates in their feeds +arxiv-cs.CR ALL Title Link Description +IACR-eprint ALL Title Link Description + +
+ +## Hobbies + +### Radio + +reddit-rtlsdr DAY Title Link + diff --git a/reveal/feedinfo b/reveal/feedinfo new file mode 100644 index 0000000..01c3ec9 --- /dev/null +++ b/reveal/feedinfo @@ -0,0 +1 @@ +https://www.revealnews.org/feed/ 60 \ No newline at end of file diff --git a/seashepherd/feedinfo b/seashepherd/feedinfo new file mode 100644 index 0000000..b8cf953 --- /dev/null +++ b/seashepherd/feedinfo @@ -0,0 +1 @@ +https://seashepherd.org/feed/ 1440 \ No newline at end of file diff --git a/securedrop/feedinfo b/securedrop/feedinfo new file mode 100644 index 0000000..7cb6b36 --- /dev/null +++ b/securedrop/feedinfo @@ -0,0 +1 @@ +https://securedrop.org/news/feed/ 1440 \ No newline at end of file diff --git a/tor-blog/feedinfo b/tor-blog/feedinfo new file mode 100644 index 0000000..9c10e18 --- /dev/null +++ b/tor-blog/feedinfo @@ -0,0 +1 @@ +https://blog.torproject.org/rss.xml 1440 \ No newline at end of file diff --git a/weather/feedinfo b/weather/feedinfo new file mode 100644 index 0000000..32ba9bb --- /dev/null +++ b/weather/feedinfo @@ -0,0 +1 @@ +https://weather.gc.ca/rss/city/bc-74_e.xml 60 diff --git a/zcash-blog/feedinfo b/zcash-blog/feedinfo new file mode 100644 index 0000000..125638e --- /dev/null +++ b/zcash-blog/feedinfo @@ -0,0 +1 @@ +https://electriccoin.co/blog/feed 1440 \ No newline at end of file diff --git a/zcash-fnd-blog/feedinfo b/zcash-fnd-blog/feedinfo new file mode 100644 index 0000000..13f233c --- /dev/null +++ b/zcash-fnd-blog/feedinfo @@ -0,0 +1 @@ +https://www.zfnd.org/feed.xml 1440 \ No newline at end of file