Initial Commit
This commit is contained in:
parent
7261b6eed5
commit
03d784d7c4
|
@ -24,3 +24,5 @@ _testmain.go
|
||||||
*.test
|
*.test
|
||||||
*.prof
|
*.prof
|
||||||
|
|
||||||
|
latest*
|
||||||
|
report.html
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
https://eprint.iacr.org/rss/rss.xml 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://www.amnesty.ca/rss.xml 1440
|
|
@ -0,0 +1 @@
|
||||||
|
http://export.arxiv.org/rss/cs.CR 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://bitcoin.org/en/rss/releases.rss 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://www.grin-forum.org/c/announce.rss 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://geti2p.net/el/feed/blog/atom 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://coindesk.com/author/lcuen/feed 60
|
|
@ -0,0 +1 @@
|
||||||
|
https://techcrunch.com/author/zack-whittaker/feed/ 60
|
|
@ -0,0 +1 @@
|
||||||
|
https://katzenpost.mixnetworks.org/blog/atom.xml 1440
|
|
@ -0,0 +1,233 @@
|
||||||
|
package main
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bufio"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"github.com/grokify/html-strip-tags-go"
|
||||||
|
"github.com/mmcdole/gofeed"
|
||||||
|
"github.com/writeas/go-strip-markdown"
|
||||||
|
"golang.org/x/net/proxy"
|
||||||
|
"io/ioutil"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"path"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func fetch(url string, cachepath string) {
|
||||||
|
fmt.Printf("Fetching [%v]\n", url)
|
||||||
|
fp := gofeed.NewParser()
|
||||||
|
feed, err := fp.ParseURL(url)
|
||||||
|
fmt.Printf("Feed %v %v\n", feed, err)
|
||||||
|
cache, _ := json.Marshal(feed)
|
||||||
|
ioutil.WriteFile(cachepath, cache, 0644)
|
||||||
|
}
|
||||||
|
|
||||||
|
func report() map[string]gofeed.Feed {
|
||||||
|
feedMap := make(map[string]gofeed.Feed)
|
||||||
|
items, _ := ioutil.ReadDir(".")
|
||||||
|
for _, item := range items {
|
||||||
|
if item.IsDir() {
|
||||||
|
//subitems, _ := ioutil.ReadDir(item.Name())
|
||||||
|
//for _, subitem := range subitems {
|
||||||
|
cachepath := path.Join(".", item.Name(), "latest")
|
||||||
|
var feed gofeed.Feed
|
||||||
|
data, err := ioutil.ReadFile(cachepath)
|
||||||
|
if err == nil {
|
||||||
|
json.Unmarshal(data, &feed)
|
||||||
|
// fmt.Printf("Feed %v\n", feed)
|
||||||
|
}
|
||||||
|
feedMap[item.Name()] = feed
|
||||||
|
//}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return feedMap
|
||||||
|
}
|
||||||
|
|
||||||
|
func download(url string, cachepath string) {
|
||||||
|
fmt.Printf("Fetching [%v]\n", url)
|
||||||
|
torDialer, err := proxy.SOCKS5("tcp", "127.0.0.1:9050", nil, proxy.Direct)
|
||||||
|
transportConfig := &http.Transport{
|
||||||
|
Dial: torDialer.Dial,
|
||||||
|
}
|
||||||
|
client := http.Client{
|
||||||
|
Transport: transportConfig,
|
||||||
|
CheckRedirect: func(r *http.Request, via []*http.Request) error {
|
||||||
|
r.URL.Opaque = r.URL.Path
|
||||||
|
return nil
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
resp, err := client.Get(url)
|
||||||
|
if err == nil {
|
||||||
|
defer resp.Body.Close()
|
||||||
|
body, _ := ioutil.ReadAll(resp.Body)
|
||||||
|
ioutil.WriteFile(cachepath, body, 0644)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func update() {
|
||||||
|
items, _ := ioutil.ReadDir(".")
|
||||||
|
for _, item := range items {
|
||||||
|
if item.IsDir() {
|
||||||
|
subitems, _ := ioutil.ReadDir(item.Name())
|
||||||
|
for _, subitem := range subitems {
|
||||||
|
if subitem.Name() == "feedinfo" {
|
||||||
|
// handle file there
|
||||||
|
filepath := path.Join(".", item.Name(), subitem.Name())
|
||||||
|
fmt.Println("Processing: " + filepath)
|
||||||
|
rawfeedinfo, _ := ioutil.ReadFile(filepath)
|
||||||
|
feedinfo := strings.Split(strings.TrimSpace(string(rawfeedinfo)), " ")
|
||||||
|
cachepath := path.Join(".", item.Name(), "latest")
|
||||||
|
info, err := os.Stat(cachepath)
|
||||||
|
if err == nil {
|
||||||
|
duration := time.Since(info.ModTime())
|
||||||
|
cron, _ := strconv.Atoi(feedinfo[1])
|
||||||
|
// If it has been greater than <cron> minutes since the last fetch, we fetch
|
||||||
|
if (time.Duration(cron) * time.Minute).Minutes() < duration.Minutes() {
|
||||||
|
fetch(feedinfo[0], cachepath)
|
||||||
|
} else {
|
||||||
|
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// First time
|
||||||
|
fetch(feedinfo[0], cachepath)
|
||||||
|
}
|
||||||
|
} else if subitem.Name() == "images" {
|
||||||
|
filepath := path.Join(".", item.Name(), subitem.Name())
|
||||||
|
file, err := os.Open(filepath)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
for scanner.Scan() {
|
||||||
|
image := scanner.Text()
|
||||||
|
imageinfo := strings.Split(image, " ")
|
||||||
|
uri, _ := url.Parse(imageinfo[0])
|
||||||
|
cachepath := path.Join(".", item.Name(), "latest-"+path.Base(uri.Path))
|
||||||
|
info, err := os.Stat(cachepath)
|
||||||
|
if err == nil {
|
||||||
|
duration := time.Since(info.ModTime())
|
||||||
|
cron, _ := strconv.Atoi(imageinfo[1])
|
||||||
|
// If it has been greater than <cron> minutes since the last fetch, we fetch
|
||||||
|
if (time.Duration(cron) * time.Minute).Minutes() < duration.Minutes() {
|
||||||
|
download(imageinfo[0], cachepath)
|
||||||
|
} else {
|
||||||
|
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// First time
|
||||||
|
download(imageinfo[0], cachepath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func processItems(format []string, items []*gofeed.Item, dateCheck time.Duration) {
|
||||||
|
num := 0
|
||||||
|
for _, item := range items {
|
||||||
|
if dateCheck == 0 {
|
||||||
|
processItem(format, *item)
|
||||||
|
num = 100
|
||||||
|
} else {
|
||||||
|
if item.PublishedParsed != nil {
|
||||||
|
if time.Since(*item.PublishedParsed) < dateCheck {
|
||||||
|
processItem(format, *item)
|
||||||
|
num++
|
||||||
|
}
|
||||||
|
} else if item.UpdatedParsed != nil {
|
||||||
|
if time.Since(*item.UpdatedParsed) < dateCheck {
|
||||||
|
processItem(format, *item)
|
||||||
|
num++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if num == 0 {
|
||||||
|
fmt.Printf("* Nothing new this %v From %v\n", format[1], format[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// stripString removes html, then potential markdown characters, and then some additional potential markdown
|
||||||
|
func stripString(input string) string {
|
||||||
|
return strings.Replace(stripmd.Strip(strip.StripTags(input)), "`", "", -1)
|
||||||
|
}
|
||||||
|
|
||||||
|
func processItem(format []string, item gofeed.Item) {
|
||||||
|
fmt.Printf("* ")
|
||||||
|
for i := 2; i < len(format); i++ {
|
||||||
|
switch format[i] {
|
||||||
|
case "Title":
|
||||||
|
fmt.Printf("%v ", stripString(item.Title))
|
||||||
|
case "Link":
|
||||||
|
fmt.Printf("[%v](%v)", stripString(item.Link), stripString(item.Link))
|
||||||
|
case "Description":
|
||||||
|
fmt.Printf("\n * %v <hr/>", strings.Replace(stripString(item.Description), "\n", "", -1))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
fmt.Printf("\n")
|
||||||
|
}
|
||||||
|
|
||||||
|
func main() {
|
||||||
|
if len(os.Args) >= 2 {
|
||||||
|
cmd := os.Args[1]
|
||||||
|
switch cmd {
|
||||||
|
case "update":
|
||||||
|
update()
|
||||||
|
case "report":
|
||||||
|
if len(os.Args) == 3 {
|
||||||
|
feeds := report()
|
||||||
|
file, err := os.Open(os.Args[2])
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
defer file.Close()
|
||||||
|
|
||||||
|
scanner := bufio.NewScanner(file)
|
||||||
|
for scanner.Scan() {
|
||||||
|
line := scanner.Text()
|
||||||
|
reportLine := strings.Split(line, " ")
|
||||||
|
if strings.HasPrefix(reportLine[0], "%") {
|
||||||
|
// Ignore, this is a comment
|
||||||
|
} else if strings.HasPrefix(reportLine[0], "#") || strings.HasPrefix(reportLine[0], "<") {
|
||||||
|
fmt.Printf("%v\n", strings.TrimSpace(line))
|
||||||
|
} else if len(reportLine) > 2 {
|
||||||
|
_, exists := feeds[reportLine[0]]
|
||||||
|
if exists {
|
||||||
|
if reportLine[1] == "ALL" {
|
||||||
|
processItems(reportLine, feeds[reportLine[0]].Items, 0)
|
||||||
|
} else if reportLine[1] == "DAY" { // Only output entries from the last Day
|
||||||
|
processItems(reportLine, feeds[reportLine[0]].Items, time.Hour*24)
|
||||||
|
} else if reportLine[1] == "WEEK" { // Only output entries from the last Week
|
||||||
|
processItems(reportLine, feeds[reportLine[0]].Items, time.Hour*24*7)
|
||||||
|
} else {
|
||||||
|
index, _ := strconv.Atoi(reportLine[1])
|
||||||
|
processItem(reportLine, *feeds[reportLine[0]].Items[index])
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
log.Fatalf("Report Template Contains Non-Existent Feed %v\n", reportLine[0])
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fmt.Printf("\n")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := scanner.Err(); err != nil {
|
||||||
|
log.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
os.Exit(0)
|
||||||
|
}
|
|
@ -0,0 +1 @@
|
||||||
|
https://web.getmonero.org/feed.xml 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://privacyinternational.org/rss.xml 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://proofofwork.news/feed/ 1440
|
|
@ -0,0 +1,2 @@
|
||||||
|
https://ocean.weather.gov/shtml/pyba01bw.gif 120
|
||||||
|
https://ocean.weather.gov/shtml/EVPN13.jpg 120
|
|
@ -0,0 +1 @@
|
||||||
|
https://www.reddit.com/r/RTLSDR.xml 1440
|
|
@ -0,0 +1,75 @@
|
||||||
|
<style> body{background:#1A111A;color:#fff;font-family:Sans;padding:0 25% 0 25%;} h2{font-size:14px;} h3{font-size:12px;} a{color:#fff;}</style>
|
||||||
|
|
||||||
|
# Daily Report
|
||||||
|
|
||||||
|
## Weather
|
||||||
|
|
||||||
|
weather 1 Title
|
||||||
|
weather 2 Title
|
||||||
|
|
||||||
|
## Satellite
|
||||||
|
|
||||||
|
<img src="./pt-reyes/latest-pyba01bw.gif" width="300" height="300"/>
|
||||||
|
<img src="./pt-reyes/latest-EVPN13.jpg" width="300" height="300"/>
|
||||||
|
|
||||||
|
<hr/>
|
||||||
|
|
||||||
|
## In The News
|
||||||
|
|
||||||
|
% Not all the journalists I follow have dedicated rss feeds available for their publication (and some don't have a publication)
|
||||||
|
journalist-zackwhittaker DAY Title Link
|
||||||
|
journalist-leighcuen DAY Title Link
|
||||||
|
reveal DAY Title Link
|
||||||
|
|
||||||
|
## Activism
|
||||||
|
|
||||||
|
seashepherd WEEK Title Link
|
||||||
|
amnesty WEEK Title Link
|
||||||
|
privacy-international WEEK Title Link
|
||||||
|
|
||||||
|
## Anonymity Blogs
|
||||||
|
|
||||||
|
tor-blog DAY Title Link
|
||||||
|
i2pblog DAY Title Link
|
||||||
|
katzenpost-blog DAY Title Link
|
||||||
|
securedrop WEEK Title Link
|
||||||
|
|
||||||
|
<hr/>
|
||||||
|
|
||||||
|
## Cryptocurrency
|
||||||
|
|
||||||
|
### General
|
||||||
|
|
||||||
|
proof-of-work WEEK Title Link
|
||||||
|
|
||||||
|
### Bitcoin
|
||||||
|
|
||||||
|
bitcoin-releases 0 Title Link
|
||||||
|
|
||||||
|
### Monero
|
||||||
|
|
||||||
|
monero WEEK Title Link
|
||||||
|
|
||||||
|
### Zcash
|
||||||
|
zcash-blog WEEK Title Link
|
||||||
|
zcash-fnd-blog WEEK Title Link
|
||||||
|
|
||||||
|
### Grin
|
||||||
|
grin-annoucements WEEK Title Link
|
||||||
|
|
||||||
|
<hr/>
|
||||||
|
|
||||||
|
## New Crypto/Security Papers
|
||||||
|
|
||||||
|
% Neither of these offer proper published dates in their feeds
|
||||||
|
arxiv-cs.CR ALL Title Link Description
|
||||||
|
IACR-eprint ALL Title Link Description
|
||||||
|
|
||||||
|
<hr/>
|
||||||
|
|
||||||
|
## Hobbies
|
||||||
|
|
||||||
|
### Radio
|
||||||
|
|
||||||
|
reddit-rtlsdr DAY Title Link
|
||||||
|
|
|
@ -0,0 +1 @@
|
||||||
|
https://www.revealnews.org/feed/ 60
|
|
@ -0,0 +1 @@
|
||||||
|
https://seashepherd.org/feed/ 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://securedrop.org/news/feed/ 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://blog.torproject.org/rss.xml 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://weather.gc.ca/rss/city/bc-74_e.xml 60
|
|
@ -0,0 +1 @@
|
||||||
|
https://electriccoin.co/blog/feed 1440
|
|
@ -0,0 +1 @@
|
||||||
|
https://www.zfnd.org/feed.xml 1440
|
Loading…
Reference in New Issue