Update scripts

.gitattributes

@@ -1,5 +1,7 @@
 * text=auto
 
+*.html linguist-language=liquid
 *.css linguist-detectable=false
-*.html linguist-detectable=false
-*.js linguist-detectable=false
+*.js linguist-detectable=false
+
+assets/* linguist-detectable=false

.github/workflows/docs.yml

@@ -32,10 +32,8 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
         with:
-          ref: ${{ inputs.ref || github.event.client_payload.ref  || env.GITHUB_REF_NAME  }}
+          ref: ${{ inputs.ref || github.event.client_payload.ref || env.GITHUB_REF_NAME }}
       - run: ./scripts/build-data.sh --remote
-      - run: |
-          ./scripts/group.sh > ./_pages/actvity.md
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 3.2

README.md

@@ -3,16 +3,51 @@
 ```sh
 ./scripts/build-data.sh --out _data
 ./scripts/build-data.sh --url https://url/to/annotations.json
-./scripts/group.sh > _pages/activity.md
+
+# locally
+./scripts/build-data.sh annotations.json
 ```
 
 ```sh
-
 bundle install    # or bundle install --local
 bundle exec jekyll build
 bundle exec jekyll serve
 ```
 
+## Building _data
+
+```sh
+# create _data/books.json
+jq -L scripts 'include "annotations"; create_book_data' annotations.json
+
+# create _data/activity.json
+jq -L scripts 'include "annotations"; create_activity_data' annotations.json
+
+# create _data/genres.json
+jq -L scripts 'include "annotations"; create_genre_data' annotations.json
+
+# create _data/words.json
+jq -L scripts 'include "annotations"; create_word_data' annotations.json
+
+# create _data/stats/bookmarks_per_month.json
+jq -L scripts 'include "annotations"; stats_bookmarks_per_month' activity.json 
+
+# create _data/stats/month.json
+jq -L scripts 'include "annotations"; stats_month' activity.json
+
+# create _data/history.json
+jq -L scripts 'include "annotations"; stats_history' activity.json 
+```
+
+## Build activity
+
+create _pages/activity.txt
+
+```sh
+jq -L scripts --slurpfile books _data/books.json \
+  'include "annotations"; stats_history_text' _data/history.json
+```
+
 # trigger update
 
 ```sh
@@ -21,5 +56,5 @@ curl -H "Accept: application/vnd.github.everest-preview+json" \
   --request POST \
   -d '{"event_type":"update_data","client_payload": {"data_url": "URL HERE", "ref": "stage" }}' \
   https://api.github.com/repos/nntrn/what-im-reading/dispatches
+```
 
-```

_config.yml

@@ -1,5 +1,5 @@
 title: what i'm reading
-description: Bookmarks from Apple Books
+description: saved excerpts of things
 author: Annie Tran
 url: https://nntrn.github.io
 baseurl: /what-im-reading

scripts/annotations.jq

@@ -1,27 +1,10 @@
 include "books";   
 
+def stop: {
+  all: ["able","about","across","after","all","almost","also","among","and","any","are","because","been","but","can","cannot","could","dear","did","does","either","else","ever","every","for","from","get","got","had","has","have","her","hers","him","his","how","however","into","its","just","least","let","like","likely","may","might","most","must","neither","nor","not","off","often","only","other","our","own","rather","said","say","says","she","should","since","some","than","that","the","their","them","then","there","these","they","this","tis","too","twas","wants","was","were","what","when","where","which","while","who","whom","why","will","with","would","yet","you","your"],
 
-#   jq -L scripts 'include "annotations"; create_book_data' annotations.json
-#   jq -L scripts 'include "annotations"; create_activity_data' annotations.json
-#   jq -L scripts 'include "annotations"; create_word_data' annotations.json
-
-#   jq -L scripts 'include "annotations"; stats_bookmarks_per_month' activity.json
-#   jq -L scripts 'include "annotations"; stats_month' activity.json
-
-def unsmart: . | gsub("[“”]";"\"") | gsub("[’‘]";"'");
-
-def remove_cites:  
-  gsub("\\[([0-9xiv]+)\\]";"";"m") | gsub("(?<q>[a-zA-Z][“”\"\\.]*)[0-9]+";.q);
-
-def format_quotes:
-  gsub("^[\\n ]+|[ \\n]+$";"")
-  | split("\n")
-  | map(
-      select(test("[a-zA-Z0-9]"))
-      | gsub("[\\t]+"; "\t")
-      | if length > 400 then  gsub("(?<=[\\S\\s]{300,500}\\.) "; "<br><br>") else . end
-    ) 
-  ;
+  compact: ["able","about","across","after","almost","also","among","because","been","cannot","could","dear","does","either","else","ever","every","from","have","hers","however","into","just","least","like","likely","might","most","must","neither","often","only","other","rather","said","says","should","since","some","than","that","their","them","then","there","these","they","this","twas","wants","were","what","when","where","which","while","whom","will","with","would","your"]
+};
 
 # functions using annotations.json ###########################################
 
@@ -60,14 +43,18 @@ def create_word_data:
       Z_PK,
       ZASSETID,
       words: (.ZANNOTATIONSELECTEDTEXT
+        | ascii_downcase
         | gsub("[.?!] (?<a>[A-Z])"; (.a|ascii_downcase);"x")
-        | gsub("\([39]|implode)"; "")
-        | gsub("[^a-zA-Z]+";" ";"x")|split(" ")
-        | map(select(length >4))|sort|group_by(.)
-        | map([.[0],length])
+        | gsub("[[:punct:]] "; " ")
+        | gsub("[^a-zA-Z]+";" ";"x")
+        | split(" ")
+        | map(select(length >4))
+        | sort
+        | group_by(.)
+        | map([(.[0]|ascii_downcase),length])
         | sort_by(.[1])
         | map(join("-"))
-        | reverse)
+        | reverse|join(" "))
     });
 
 def create_genre_data: 
@@ -80,6 +67,7 @@ def create_genre_data:
     });
 
 # functions using activity.json ###############################################
+
 def stats_bookmarks_per_month:
   sort_by(.created) 
   | map(. + {groupby_label: (.created|fromdate|strftime("%b %Y"))}) 
@@ -95,4 +83,65 @@ def stats_month:
       saved_count: length, 
       saved: (group_by(.assetid)|map({(.[0].assetid): length})|add)  
       } 
-  }) | add;
+  }) | add;
+
+def stats_history:
+  map((.created|strptime("%Y-%m-%dT%H:%M:%SZ")|mktime) as $dt | 
+    { 
+      id,assetid,created,
+      date: $dt, 
+      ymd: ($dt|strftime("%Y-%m-%d")), 
+      month: ($dt|strftime("%b %Y")) 
+    })
+  | group_by(.month)
+  | map({ 
+      month: .[0].month, 
+      sortdt: (.[0].date|strftime("%Y %m")),
+      month_count: length,
+      activity: ( 
+        group_by(.ymd)
+        | reverse
+        | map({
+            date: .[0].ymd, 
+            date_count: length,
+            books: (map(.assetid)|group_by(.) | map({  assetid: .[0], book_count: length }))
+        })
+      )
+    })
+  | sort_by(.sortdt)
+  | reverse;
+
+# jq -r -L scripts --slurpfile books _data/books.json 'include "annotations"; stats_history_text' _data/history.json
+def stats_history_text:
+  if ($books|length) > 0 then
+  ( [  
+      # title
+      "Reading activity", "="*16,
+      "", 
+      "*** denotes date when book was started",
+      "",
+      # create markdown file showing activity from _data/history.json
+      # index $books to get date book was started
+      ($books[] | map(. + {cdate: (.created|strptime("%Y-%m-%dT%H:%M:%SZ")|mktime|strftime("%Y-%m-%d"))}) |INDEX(.[];.assetid)) as $asset | map([
+          .month, 
+          ("-" * (.month|length)), "",
+          (.activity | map( .date as $d2 | [
+            "  * **\(.date)**", "", 
+            (
+              .books | map((if .book_count > 1 then "annotations" else "annotation" end) as $noun
+                | (if $asset[.assetid].cdate == $d2 then " ***" else "" end) as $starttext
+                | "     - [\($asset[.assetid].title|gsub("[\\[\\]]";""))][\(.assetid)] - \(.book_count) \($noun)\($starttext)" 
+                ) 
+            ),
+            ""
+            ])
+          ), ""
+      ]), 
+      # create link reference from _data/books.json
+      ($books[]|sort_by(.permalink)|map("[\(.assetid)]:\t\(.permalink)"))
+    ] 
+    | flatten | join("\n") )
+  else  
+    error("path to _data/books.json was not passed")
+  end
+;

scripts/books.jq

@@ -1,3 +1,8 @@
+# def stopwords: {
+#   all: ["able","about","across","after","all","almost","also","among","and","any","are","because","been","but","can","cannot","could","dear","did","does","either","else","ever","every","for","from","get","got","had","has","have","her","hers","him","his","how","however","into","its","just","least","let","like","likely","may","might","most","must","neither","nor","not","off","often","only","other","our","own","rather","said","say","says","she","should","since","some","than","that","the","their","them","then","there","these","they","this","tis","too","twas","wants","was","were","what","when","where","which","while","who","whom","why","will","with","would","yet","you","your"],
+#   compact: ["able","about","across","after","almost","also","among","because","been","cannot","could","dear","does","either","else","ever","every","from","have","hers","however","into","just","least","like","likely","might","most","must","neither","often","only","other","rather","said","says","should","since","some","than","that","their","them","then","there","these","they","this","twas","wants","were","what","when","where","which","while","whom","will","with","would","your"]
+# }
+
 def squo: [39]|implode;
 def lpad(n): tostring | if (n > length) then ((n - length) * "0") + . else . end;
 def squote($text): [squo,$text,squo]|join("");
@@ -22,13 +27,6 @@ def epublocation($cfi):
   | split("-")|.[1:]
   | map(select(length < 5)|tonumber);
 
-def remove_citations($text):
-  $text 
-    | gsub("(?<a>[^0-9\\$,]{3})[0-9]{1,2}(?<b>[^0-9%\\.,]{2})"; .a +.b; "x")
-    | gsub("(?<a>[^0-9]{3})[0-9]{1,2}([\\s]+)?$"; .a +.b; "xs");
-
-def remove_citations: remove_citations(.);
-
 def split_long_title($text):
   $text
   | split("\\s?[(:)]\\s?";"x")
@@ -52,6 +50,19 @@ def get_author_slug($s):
   | ascii_downcase
   ;
 
+def remove_cites:  
+  gsub("\\[([0-9xiv]+)\\]";"";"m") | gsub("(?<q>[a-zA-Z][“”\"\\.]*)[0-9]+";.q);
+
+def format_quotes:
+  gsub("^[\\n ]+|[ \\n]+$";"")
+  | split("\n")
+  | map(
+      select(test("[a-zA-Z0-9]"))
+      | gsub("[\\t]+"; "\t")
+      | if length > 400 then  gsub("(?<=[\\S\\s]{300,500}\\.) "; "\n\n") else . end
+    ) 
+  ;
+
 def identify_subsection:
  . as $string |
   if test("[cC][onclusi]{2,}") then "Conclusion"