sarah
/
kdb
2
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Formatting

This commit is contained in:
Sarah Jamie Lewis 2019-11-11 19:07:25 -08:00
parent 37bebe271e
commit 6543f89843
4 changed files with 51 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored Normal file
View File

@ -0,0 +1,4 @@
tmp.dot
tmp.png
tmp.svg
.idea/

20
citing/cite.go
View File

@ -14,9 +14,9 @@ import (
"strings"
)
func ExtractPDF(extract string) (string,string,string,*bibtex.BibEntry) {
func ExtractPDF(extract string) (string, string, string, *bibtex.BibEntry) {
log.Infof("Extracting..." + extract)
parts := strings.Split(extract,";")
parts := strings.Split(extract, ";")
if len(parts) >= 3 {
args := strings.Split(parts[1], ":")
@ -24,7 +24,7 @@ func ExtractPDF(extract string) (string,string,string,*bibtex.BibEntry) {
id := sha512.Sum512([]byte(src))
fetch.Fetch(src, hex.EncodeToString(id[:]))
cachepath := path.Join("cache/", hex.EncodeToString(id[:]))
args = append(args, "-singlefile", "-png", cachepath, "tmp")
args = append(args, "-singlefile", "-png", cachepath, "tmp")
subProcess := exec.Command("pdftoppm", args...)
log.Infof("Running.. %v", subProcess.Args, subProcess.String())
err := subProcess.Run()
@ -36,22 +36,22 @@ func ExtractPDF(extract string) (string,string,string,*bibtex.BibEntry) {
bt := FindReference(src)
title := "From " + src
if bt != nil {
title = bt.Fields["title"].String() + " by " + bt.Fields["author"].String() + " (" + bt.Fields["year"] .String()+ ")"
title = bt.Fields["title"].String() + " by " + bt.Fields["author"].String() + " (" + bt.Fields["year"].String() + ")"
}
return title,link, "<img class=\"source\" id=\"" + link + "\" src=\"data:image/png;base64, " + base64.StdEncoding.EncodeToString(data) + "\"/>",bt
return title, link, "<img class=\"source\" id=\"" + link + "\" src=\"data:image/png;base64, " + base64.StdEncoding.EncodeToString(data) + "\"/>", bt
}
return "","","",nil
return "", "", "", nil
}
func FindReference(url string) *bibtex.BibEntry{
file,err := os.Open("./data/references.kdb")
func FindReference(url string) *bibtex.BibEntry {
file, err := os.Open("./data/references.kdb")
if err == nil {
bt,err := bibtex.Parse(file)
bt, err := bibtex.Parse(file)
if err != nil {
log.Errorf("%v", err)
}
for _,e := range bt.Entries {
for _, e := range bt.Entries {
if e.Fields["url"].String() == url {
return e
}

7
fetch/fetch.go
View File

@ -8,6 +8,7 @@ import (
"os"
"path"
)
func makeTorifiedClient() *http.Client {
torDialer, err := proxy.SOCKS5("tcp", "127.0.0.1:9050", nil, proxy.Direct)
if err != nil {
@ -29,12 +30,12 @@ func Fetch(url string, cacheid string) []byte {
log.Infof("Fetching..." + url)
cachepath := path.Join("cache/", cacheid)
if _, err := os.Stat(cachepath); !os.IsNotExist(err) {
data,_ := ioutil.ReadFile(cachepath)
data, _ := ioutil.ReadFile(cachepath)
return data
}
client := makeTorifiedClient()
resp,err:= client.Get(url)
resp, err := client.Get(url)
if err == nil {
defer resp.Body.Close()
@ -45,5 +46,3 @@ func Fetch(url string, cacheid string) []byte {
log.Errorf("Error fetching: %v", err)
return []byte{}
}

73
kdb.go
View File

@ -1,14 +1,13 @@
package main
import (
"errors"
"fmt"
"git.openprivacy.ca/openprivacy/libricochet-go/log"
"git.openprivacy.ca/sarah/kdb/citing"
"github.com/nickng/bibtex"
"html/template"
"io/ioutil"
"git.openprivacy.ca/openprivacy/libricochet-go/log"
"net/http"
"os"
"os/exec"
@ -19,10 +18,10 @@ import (
)
type Page struct {
Title string
Body template.HTML
BodyOrig string
Concept template.HTML
Title string
Body template.HTML
BodyOrig string
Concept template.HTML
References map[string]*bibtex.BibEntry
}
@ -76,15 +75,15 @@ func loadPage(title string) (*Page, error) {
if hasPage(word) {
body = strings.ReplaceAll(body, word, fmt.Sprintf("<a href='/view/%v/'>%v</a>", normalize(word), normalize(word)))
}
if strings.HasPrefix(word,"cite;") {
if strings.HasPrefix(word, "cite;") {
log.Infof("Extracting Citation")
cite,link,img,rec := citing.ExtractPDF(word)
cite, link, img, rec := citing.ExtractPDF(word)
if rec != nil {
if references[rec.CiteName] == nil {
references[rec.CiteName] = rec
}
}
body = strings.ReplaceAll(body, word, fmt.Sprintf("<a title=\"%v\" href=\"javascript:showsource('%v')\">[X]</a>%v",cite,link,img))
body = strings.ReplaceAll(body, word, fmt.Sprintf("<a title=\"%v\" href=\"javascript:showsource('%v')\">[X]</a>%v", cite, link, img))
}
if strings.HasPrefix(word, "http") {
body = strings.ReplaceAll(body, word, fmt.Sprintf("<a onmouseover=\"javascript:showsource('%v')\" href='%v'></a>", word, word, word))
@ -96,7 +95,7 @@ func loadPage(title string) (*Page, error) {
}
}
return &Page{Title: title, Body: template.HTML(body), BodyOrig: string(bodyb), Concept: concept, References:references}, nil
return &Page{Title: title, Body: template.HTML(body), BodyOrig: string(bodyb), Concept: concept, References: references}, nil
}
func normalize(word string) string {
@ -104,7 +103,7 @@ func normalize(word string) string {
word = word[7:]
} else if strings.HasPrefix(word, "subtype:") {
word = word[8:]
} else if strings.HasPrefix(word, "instance:") {
} else if strings.HasPrefix(word, "instance:") {
word = word[9:]
}
word = strings.TrimSpace(word)
@ -124,7 +123,6 @@ func viewHandler(w http.ResponseWriter, r *http.Request) {
renderTemplate(w, "view", p)
}
func saveHandler(w http.ResponseWriter, r *http.Request) {
title, err := getTitle(w, r)
if err != nil {
@ -157,15 +155,15 @@ func get_links(name string) (links []string) {
func construct_graph(word string) string {
seen := make(map[string]bool)
graph := "digraph {\n ratio=\"compress\";size=\"8,4!\";splines=true;\""+word+"\";\n"
graph := "digraph {\n ratio=\"compress\";size=\"8,4!\";splines=true;\"" + word + "\";\n"
graph = construct_graph_sub(graph, word, seen, 1)
graph += "}"
ioutil.WriteFile("tmp.dot", []byte(graph), 0640)
subProcess := exec.Command("dot", "-Tsvg", "-otmp.svg","-Nfontname=Chilanka", "tmp.dot")
subProcess := exec.Command("dot", "-Tsvg", "-otmp.svg", "-Nfontname=Chilanka", "tmp.dot")
subProcess.Run()
subProcess.Wait()
data, _ := ioutil.ReadFile("tmp.svg")
return "<div class=\"conceptgraph\">"+strings.ReplaceAll(string(data), `<?xml version="1.0" encoding="UTF-8" standalone="no"?>
return "<div class=\"conceptgraph\">" + strings.ReplaceAll(string(data), `<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN"
"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">`, "")
}
@ -181,7 +179,7 @@ func construct_graph_sub(graph string, word string, seen map[string]bool, depth
if depth == 1 {
col = "fillcolor=\"#b09cbc\""
}
graph += fmt.Sprintf("\"%v\" [href=\"/view/%v\",%v,style=filled];\n", normalize(word), normalize(word),col)
graph += fmt.Sprintf("\"%v\" [href=\"/view/%v\",%v,style=filled];\n", normalize(word), normalize(word), col)
}
seen[normalize(word)] = true
@ -189,7 +187,7 @@ func construct_graph_sub(graph string, word string, seen map[string]bool, depth
graph += "# " + word + "\n"
for _, link := range links {
if strings.HasPrefix(word, "typeof:") || strings.HasPrefix(word, "instance:") || strings.HasPrefix(word, "subtype:") {
if strings.HasPrefix(word, "typeof:") || strings.HasPrefix(word, "instance:") || strings.HasPrefix(word, "subtype:") {
word = normalize(word)
}
@ -197,35 +195,32 @@ func construct_graph_sub(graph string, word string, seen map[string]bool, depth
graph += fmt.Sprintf("\"%v\" [href=\"/view/%v\"];\n", normalize(link), normalize(link))
}
if normalize(word) != normalize(link) && (!seen[normalize(word)+"++"+normalize(link)] || !seen[normalize(link)+"++"+normalize(word)]) {
if strings.HasPrefix(link, "typeof:") {
graph += fmt.Sprintf("\"%v\" -> \"%v\" [style=dotted]\n", link[7:], word)
seen[word+"++"+link[7:]] = true
seen[link[7:]+"++"+word] = true
} else if strings.HasPrefix(link, "subtype:") {
graph += fmt.Sprintf("\"%v\" -> \"%v\" [style=dotted]\n", word, link[8:])
seen[word+"++"+link[8:]] = true
seen[link[8:]+"++"+word] = true
} else if strings.HasPrefix(link, "instance:") {
graph += fmt.Sprintf("\"%v\" -> \"%v\" [style=dotted]\n", word, link[9:])
seen[word+"++"+link[9:]] = true
seen[link[9:]+"++"+word] = true
} else {
graph += fmt.Sprintf("\"%v\" -> \"%v\" []\n", word, link)
}
seen[normalize(word)+"++"+normalize(link)] = true
seen[normalize(link)+"++"+normalize(word)] = true
if normalize(word) != normalize(link) && (!seen[normalize(word)+"++"+normalize(link)] || !seen[normalize(link)+"++"+normalize(word)]) {
if strings.HasPrefix(link, "typeof:") {
graph += fmt.Sprintf("\"%v\" -> \"%v\" [style=dotted]\n", link[7:], word)
seen[word+"++"+link[7:]] = true
seen[link[7:]+"++"+word] = true
} else if strings.HasPrefix(link, "subtype:") {
graph += fmt.Sprintf("\"%v\" -> \"%v\" [style=dotted]\n", word, link[8:])
seen[word+"++"+link[8:]] = true
seen[link[8:]+"++"+word] = true
} else if strings.HasPrefix(link, "instance:") {
graph += fmt.Sprintf("\"%v\" -> \"%v\" [style=dotted]\n", word, link[9:])
seen[word+"++"+link[9:]] = true
seen[link[9:]+"++"+word] = true
} else {
graph += fmt.Sprintf("\"%v\" -> \"%v\" []\n", word, link)
}
//}
seen[normalize(word)+"++"+normalize(link)] = true
seen[normalize(link)+"++"+normalize(word)] = true
}
//}
if !seen[link] {
graph = construct_graph_sub(graph, link, seen, depth-1)
}
}
return graph