diff --git a/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsFrom.upToPage..st b/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsFrom.upToPage..st new file mode 100644 index 0000000..348f37c --- /dev/null +++ b/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsFrom.upToPage..st @@ -0,0 +1,33 @@ +accessing +collectRawTweetsFrom: anUrl upToPage: anInteger + + | pagesDict response customQuery | + pagesDict := self getPagesContentsFrom: anUrl upTo: anInteger. + response := TweetsCollection new. + customQuery := Dictionary new + at: 'parameters' put: pagesDict keys; + at: 'date' put: DateAndTime now; + yourself. + response query: customQuery. + pagesDict keysAndValuesDo: [ :key :rawTweets | | temp | + temp := (rawTweets xpath: '//div[@class="timeline-item "]') asOrderedCollection + collect: [ :xmlElement | xmlElement postCopy ]. + temp do: [ :tweet | | tempTweet | + tempTweet := Tweet new fromNitterHtmlItem: tweet. + tempTweet metadata + at: DateAndTime now asString put: key; + yourself. + response add: tempTweet. + ] + ]. + response messages: (response messages select: [ :tweet | tweet isNotNil ]). + response messages doWithIndex: [ :tweet :i | + | current previous | + current := response messages at: i. + i < response lastIndex ifTrue: [ + previous := response messages at: i + 1. + current timelines + at: self userName put: previous id; + yourself ]]. + ^ response. + \ No newline at end of file diff --git a/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsUpToPage..st b/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsUpToPage..st index 304d9fe..8459da9 100644 --- a/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsUpToPage..st +++ b/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsUpToPage..st @@ -1,33 +1,5 @@ accessing collectRawTweetsUpToPage: anInteger - | pagesDict response customQuery | - pagesDict := self getPagesContentsUpto: anInteger. - response := TweetsCollection new. - customQuery := Dictionary new - at: 'parameters' put: pagesDict keys; - at: 'date' put: DateAndTime now; - yourself. - response query: customQuery. - pagesDict keysAndValuesDo: [ :key :rawTweets | | temp | - temp := (rawTweets xpath: '//div[@class="timeline-item "]') asOrderedCollection - collect: [ :xmlElement | xmlElement postCopy ]. - temp do: [ :tweet | | tempTweet | - tempTweet := Tweet new fromNitterHtmlItem: tweet. - tempTweet metadata - at: DateAndTime now asString put: key; - yourself. - response add: tempTweet. - ] - ]. - response messages: (response messages select: [ :tweet | tweet isNotNil ]). - response messages doWithIndex: [ :tweet :i | - | current previous | - current := response messages at: i. - i < response lastIndex ifTrue: [ - previous := response messages at: i + 1. - current timelines - at: self userName put: previous id; - yourself ]]. - ^ response. + ^ self collectRawTweetsFrom: self userNameLinkWithReplies upToPage: anInteger \ No newline at end of file diff --git a/Socialmetrica.package/NitterUser.class/instance/documentTree.st b/Socialmetrica.package/NitterUser.class/instance/documentTree.st index a6ae6e5..40fb2a1 100644 --- a/Socialmetrica.package/NitterUser.class/instance/documentTree.st +++ b/Socialmetrica.package/NitterUser.class/instance/documentTree.st @@ -1,3 +1,3 @@ operation documentTree - ^ self documentTreeFor: (self userNameLink, '/with_replies') \ No newline at end of file + ^ self documentTreeFor: self userNameLinkWithReplies \ No newline at end of file diff --git a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.Upto..st b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.upTo..st similarity index 93% rename from Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.Upto..st rename to Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.upTo..st index fa8665a..e39ecd2 100644 --- a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.Upto..st +++ b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFrom.upTo..st @@ -1,5 +1,5 @@ accessing -getPagesContentsFrom: anURL Upto: anInteger +getPagesContentsFrom: anURL upTo: anInteger "I retroactively get all pages contents until a specified page number. TO DO: should this be splitted back to two methods, one getting the page urls and other its content? diff --git a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFromOldestUpto..st b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFromOldestUpto..st index 8e9d9f4..c80a6c4 100644 --- a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFromOldestUpto..st +++ b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsFromOldestUpto..st @@ -1,4 +1,4 @@ accessing getPagesContentsFromOldestUpto: anInteger - ^ self getPagesContentsFrom: ((self oldestTweet metadata select: [ :item | item isString and: [ item beginsWith: 'https://' ]]) values first) Upto: anInteger \ No newline at end of file + ^ self getPagesContentsFrom: ((self oldestTweet metadata select: [ :item | item isString and: [ item beginsWith: 'https://' ]]) values first) upTo: anInteger \ No newline at end of file diff --git a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsUpto..st b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsUpto..st index fe41a0e..57f93e4 100644 --- a/Socialmetrica.package/NitterUser.class/instance/getPagesContentsUpto..st +++ b/Socialmetrica.package/NitterUser.class/instance/getPagesContentsUpto..st @@ -5,4 +5,4 @@ getPagesContentsUpto: anInteger TO DO: should this be splitted back to two methods, one getting the page urls and other its content? or do we always be getting the cursor urls and its contents all the time. [ ] Benchmark alternative approaches." - ^ self getPagesContentsFrom: (self userNameLink, '/with_replies') Upto: anInteger \ No newline at end of file + ^ self getPagesContentsFrom: self userNameLinkWithReplies upTo: anInteger \ No newline at end of file diff --git a/Socialmetrica.package/NitterUser.class/instance/userNameLinkWithReplies.st b/Socialmetrica.package/NitterUser.class/instance/userNameLinkWithReplies.st new file mode 100644 index 0000000..a597f9d --- /dev/null +++ b/Socialmetrica.package/NitterUser.class/instance/userNameLinkWithReplies.st @@ -0,0 +1,3 @@ +operation +userNameLinkWithReplies + ^ self userNameLink, '/with_replies' \ No newline at end of file