From 06b0d5748d651321fbaef1d34e311a14cc76c1e3 Mon Sep 17 00:00:00 2001 From: ruidajo Date: Wed, 27 Apr 2022 23:51:53 -0500 Subject: [PATCH] Creating download tweets from oldest up to a page, modifying histograms exporters and external word cloud without mask. --- .../collectRawTweetsFromOldestUpToPage..st | 33 +++++++++++++++++++ .../instance/exportQuotesHistogram.st | 5 +-- .../exportQuotesHistogramWithBars..st | 5 +-- .../exportRepliesHistogramWithBars..st | 4 +-- .../instance/exportRetweetsHistogram.st | 5 +-- .../exportRetweetsHistogramWithBars..st | 5 +-- .../exportTweetsHistogramWithBars..st | 4 +-- .../instance/externalWordCloud.st | 5 +-- .../instance/getPagesContentsFrom.Upto..st | 19 +++++++++++ .../getPagesContentsFromOldestUpto..st | 4 +++ .../instance/getPagesContentsUpto..st | 13 +------- 11 files changed, 76 insertions(+), 26 deletions(-) create mode 100644 Socialmetrica.package/NitterUser.class/instance/collectRawTweetsFromOldestUpToPage..st create mode 100644 Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.Upto..st create mode 100644 Socialmetrica.package/NitterUser.class/instance/getPagesContentsFromOldestUpto..st diff --git a/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsFromOldestUpToPage..st b/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsFromOldestUpToPage..st new file mode 100644 index 0000000..d91e317 --- /dev/null +++ b/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsFromOldestUpToPage..st @@ -0,0 +1,33 @@ +accessing +collectRawTweetsFromOldestUpToPage: anInteger + + | pagesDict response customQuery | + pagesDict := self getPagesContentsFromOldestUpto: anInteger. + response := TweetsCollection new. + customQuery := Dictionary new + at: 'parameters' put: pagesDict keys; + at: 'date' put: DateAndTime now; + yourself. + response query: customQuery. + pagesDict keysAndValuesDo: [ :key :rawTweets | | temp | + temp := (rawTweets xpath: '//div[@class="timeline-item "]') asOrderedCollection + collect: [ :xmlElement | xmlElement postCopy ]. + temp do: [ :tweet | | tempTweet | + tempTweet := Tweet new fromNitterHtmlItem: tweet. + tempTweet metadata + at: DateAndTime now asString put: key; + yourself. + response add: tempTweet. + ] + ]. + response messages: (response messages select: [ :tweet | tweet isNotNil ]). + response messages doWithIndex: [ :tweet :i | + | current previous | + current := response messages at: i. + i < response lastIndex ifTrue: [ + previous := response messages at: i + 1. + current timelines + at: self userName put: previous id; + yourself ]]. + ^ response. + \ No newline at end of file diff --git a/Socialmetrica.package/NitterUser.class/instance/exportQuotesHistogram.st b/Socialmetrica.package/NitterUser.class/instance/exportQuotesHistogram.st index e7b0754..fb3f10d 100644 --- a/Socialmetrica.package/NitterUser.class/instance/exportQuotesHistogram.st +++ b/Socialmetrica.package/NitterUser.class/instance/exportQuotesHistogram.st @@ -5,6 +5,7 @@ exportQuotesHistogram quotesOccurrences := self quotesSortedByOccurrences. labels := quotesOccurrences keys. + labels := labels collect: [ :profiles | ('@', profiles) ]. quotesHistogram := RSChart new. quotesHistogram extent: 800@200. diagram := RSBarPlot new @@ -15,11 +16,11 @@ exportQuotesHistogram quotesHistogram addDecoration: (RSHorizontalTick new fromNames: labels; labelRotation: 0; - fontSize: 80 /quotesOccurrences size; + fontSize: 72 /quotesOccurrences size; yourself). quotesHistogram addDecoration: (RSVerticalTick new integer; - fontSize: 80 /quotesOccurrences size). + fontSize: 72 /quotesOccurrences size). quotesHistogram build. quotesHistogram canvas pdfExporter diff --git a/Socialmetrica.package/NitterUser.class/instance/exportQuotesHistogramWithBars..st b/Socialmetrica.package/NitterUser.class/instance/exportQuotesHistogramWithBars..st index e852c45..d57d262 100644 --- a/Socialmetrica.package/NitterUser.class/instance/exportQuotesHistogramWithBars..st +++ b/Socialmetrica.package/NitterUser.class/instance/exportQuotesHistogramWithBars..st @@ -10,6 +10,7 @@ exportQuotesHistogramWithBars: aNumberOfBars quotes removeKeys: keysToRemove. labels := quotes keys. + labels := labels collect: [ :profiles | ('@', profiles) ]. quotesHistogram := RSChart new. quotesHistogram extent: 800@200. diagram := RSBarPlot new @@ -20,11 +21,11 @@ exportQuotesHistogramWithBars: aNumberOfBars quotesHistogram addDecoration: (RSHorizontalTick new fromNames: labels; labelRotation: 0; - fontSize: 80 /quotes size; + fontSize: 72 /quotes size; yourself). quotesHistogram addDecoration: (RSVerticalTick new integer; - fontSize: 80 /quotes size). + fontSize: 72 /quotes size). quotesHistogram build. quotesHistogram canvas pdfExporter diff --git a/Socialmetrica.package/NitterUser.class/instance/exportRepliesHistogramWithBars..st b/Socialmetrica.package/NitterUser.class/instance/exportRepliesHistogramWithBars..st index 06c1d5c..2081562 100644 --- a/Socialmetrica.package/NitterUser.class/instance/exportRepliesHistogramWithBars..st +++ b/Socialmetrica.package/NitterUser.class/instance/exportRepliesHistogramWithBars..st @@ -15,11 +15,11 @@ exportRepliesHistogramWithBars: aNumberOfBars tweetsHistogram addDecoration: (RSHorizontalTick new fromNames: tweetsByTimeSpan keys; labelRotation: 0; - fontSize: 80 /aNumberOfBars; + fontSize: 72 /aNumberOfBars; yourself). tweetsHistogram addDecoration: (RSVerticalTick new integer; - fontSize: 80 /aNumberOfBars). + fontSize: 72 /aNumberOfBars). tweetsHistogram build. tweetsHistogram canvas pngExporter zoomToShapes; diff --git a/Socialmetrica.package/NitterUser.class/instance/exportRetweetsHistogram.st b/Socialmetrica.package/NitterUser.class/instance/exportRetweetsHistogram.st index 440de3a..28d306f 100644 --- a/Socialmetrica.package/NitterUser.class/instance/exportRetweetsHistogram.st +++ b/Socialmetrica.package/NitterUser.class/instance/exportRetweetsHistogram.st @@ -6,6 +6,7 @@ exportRetweetsHistogram retweetsOccurrences := self retweetsSortedByOccurrences. retweetColor := (Color r:(217/255) g:(56/255) b: (124/255)). labels := retweetsOccurrences keys. + labels := labels collect: [ :profiles | ('@', profiles) ]. retweetsHistogram := RSChart new. retweetsHistogram extent: 800@200. diagram := RSBarPlot new @@ -16,11 +17,11 @@ exportRetweetsHistogram retweetsHistogram addDecoration: (RSHorizontalTick new fromNames: labels; labelRotation: 0; - fontSize: 80 /retweetsOccurrences size; + fontSize: 72 /retweetsOccurrences size; yourself). retweetsHistogram addDecoration: (RSVerticalTick new integer; - fontSize: 80 /retweetsOccurrences size). + fontSize: 72 /retweetsOccurrences size). retweetsHistogram build. retweetsHistogram canvas pdfExporter diff --git a/Socialmetrica.package/NitterUser.class/instance/exportRetweetsHistogramWithBars..st b/Socialmetrica.package/NitterUser.class/instance/exportRetweetsHistogramWithBars..st index a522892..b7d84cb 100644 --- a/Socialmetrica.package/NitterUser.class/instance/exportRetweetsHistogramWithBars..st +++ b/Socialmetrica.package/NitterUser.class/instance/exportRetweetsHistogramWithBars..st @@ -10,6 +10,7 @@ exportRetweetsHistogramWithBars: aNumberOfBars retweets removeKeys: keysToRemove. labels := retweets keys. + labels := labels collect: [ :profiles | ('@', profiles) ]. retweetsHistogram := RSChart new. retweetsHistogram extent: 800@200. diagram := RSBarPlot new @@ -20,11 +21,11 @@ exportRetweetsHistogramWithBars: aNumberOfBars retweetsHistogram addDecoration: (RSHorizontalTick new fromNames: labels; labelRotation: 0; - fontSize: 80 /retweets size; + fontSize: 72 /retweets size; yourself). retweetsHistogram addDecoration: (RSVerticalTick new integer; - fontSize: 80 /retweets size). + fontSize: 72 /retweets size). retweetsHistogram build. retweetsHistogram canvas pdfExporter diff --git a/Socialmetrica.package/NitterUser.class/instance/exportTweetsHistogramWithBars..st b/Socialmetrica.package/NitterUser.class/instance/exportTweetsHistogramWithBars..st index 595b502..90402fc 100644 --- a/Socialmetrica.package/NitterUser.class/instance/exportTweetsHistogramWithBars..st +++ b/Socialmetrica.package/NitterUser.class/instance/exportTweetsHistogramWithBars..st @@ -15,11 +15,11 @@ exportTweetsHistogramWithBars: aNumberOfBars tweetsHistogram addDecoration: (RSHorizontalTick new fromNames: tweetsByTimeSpan keys; labelRotation: 0; - fontSize: 80 /aNumberOfBars; + fontSize: 72 /aNumberOfBars; yourself). tweetsHistogram addDecoration: (RSVerticalTick new integer; - fontSize: 80 /aNumberOfBars). + fontSize: 72 /aNumberOfBars). tweetsHistogram build. tweetsHistogram canvas pngExporter zoomToShapes; diff --git a/Socialmetrica.package/NitterUser.class/instance/externalWordCloud.st b/Socialmetrica.package/NitterUser.class/instance/externalWordCloud.st index 3d4de6c..7f4c20a 100644 --- a/Socialmetrica.package/NitterUser.class/instance/externalWordCloud.st +++ b/Socialmetrica.package/NitterUser.class/instance/externalWordCloud.st @@ -16,8 +16,9 @@ externalWordCloud '--height' . '357' . '--background' . 'white' . '--mode' . 'RGBA' . - '--stopwords' . '../../../commons/stopwords-es.txt' . - '--mask' . '../../../commons/nube-mascara.jpg'}; + '--stopwords' . '../../../commons/stopwords-es.txt'. + "'--mask' . '../../../commons/nube-mascara.jpg'" + }; workingDirectory: self folder fullName; redirectStdout; redirectStderr; diff --git a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.Upto..st b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.Upto..st new file mode 100644 index 0000000..fa8665a --- /dev/null +++ b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.Upto..st @@ -0,0 +1,19 @@ +accessing +getPagesContentsFrom: anURL Upto: anInteger + "I retroactively get all pages contents until a specified page number. + + TO DO: should this be splitted back to two methods, one getting the page urls and other its content? + or do we always be getting the cursor urls and its contents all the time. + [ ] Benchmark alternative approaches." + | response nextPageLink previousPageLink | + + response := OrderedDictionary new. + response at: anURL put: (self documentTreeFor: anURL). + previousPageLink := anURL. + anInteger - 1 timesRepeat: [ | pageCursor | + pageCursor := self pageCursorFor:previousPageLink. + nextPageLink := self userNameLink, '/with_replies', pageCursor keys first. + response at: nextPageLink put: (XMLHTMLParser parse:nextPageLink asUrl retrieveContents). + previousPageLink := nextPageLink + ]. + ^ response \ No newline at end of file diff --git a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFromOldestUpto..st b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFromOldestUpto..st new file mode 100644 index 0000000..448b940 --- /dev/null +++ b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFromOldestUpto..st @@ -0,0 +1,4 @@ +accessing +getPagesContentsFromOldestUpto: anInteger + + ^ self getPagesContentsFrom: ((self messages oldest metadata select: [ :item | item beginsWith: 'https://' ]) values first) Upto: anInteger \ No newline at end of file diff --git a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsUpto..st b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsUpto..st index 1d9e806..fe41a0e 100644 --- a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsUpto..st +++ b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsUpto..st @@ -5,15 +5,4 @@ getPagesContentsUpto: anInteger TO DO: should this be splitted back to two methods, one getting the page urls and other its content? or do we always be getting the cursor urls and its contents all the time. [ ] Benchmark alternative approaches." - | response nextPageLink previousPageLink | - - response := OrderedDictionary new. - response at: (self userNameLink, '/with_replies') put: self documentTree. - previousPageLink := (self userNameLink, '/with_replies'). - anInteger - 1 timesRepeat: [ | pageCursor | - pageCursor := self pageCursorFor:previousPageLink. - nextPageLink := self userNameLink, '/with_replies', pageCursor keys first. - response at: nextPageLink put: (XMLHTMLParser parse:nextPageLink asUrl retrieveContents). - previousPageLink := nextPageLink - ]. - ^ response \ No newline at end of file + ^ self getPagesContentsFrom: (self userNameLink, '/with_replies') Upto: anInteger \ No newline at end of file