diff --git a/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsForPages..st b/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsForPages..st index 7739892..5e961ac 100644 --- a/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsForPages..st +++ b/Socialmetrica.package/NitterUser.class/instance/collectRawTweetsForPages..st @@ -1,18 +1,32 @@ accessing collectRawTweetsForPages: anInteger - | pagesDict response | + | pagesDict response customQuery | pagesDict := self getPagesContentsUpto: anInteger. response := TweetsCollection new. - response query: pagesDict keys. + customQuery := Dictionary new + at: 'parameters' put: pagesDict keys; + at: 'date' put: DateAndTime now; + yourself. + response query: customQuery. pagesDict keysAndValuesDo: [ :key :rawTweets | | temp | temp := (rawTweets xpath: '//div[@class="timeline-item "]') asOrderedCollection collect: [ :xmlElement | xmlElement postCopy ]. temp do: [ :tweet | | tempTweet | tempTweet := Tweet new fromNitterHtmlItem: tweet. - tempTweet queries add: key. + tempTweet metadata + at: DateAndTime now put:key; + yourself. response tweets add: tempTweet. ] ]. + response tweets doWithIndex: [ :tweet :i | + | current previous | + current := response tweets at: i. + i < response tweets size ifTrue: [ + previous := response tweets at: i + 1. + current timelines + at: self userName put: previous id; + yourself ]]. ^ response. \ No newline at end of file diff --git a/Socialmetrica.package/Tweet.class/class/reStoreDefinition.st b/Socialmetrica.package/Tweet.class/class/reStoreDefinition.st index 4c1421f..6ed4e8d 100644 --- a/Socialmetrica.package/Tweet.class/class/reStoreDefinition.st +++ b/Socialmetrica.package/Tweet.class/class/reStoreDefinition.st @@ -1,10 +1,11 @@ accessing reStoreDefinition ^ super reStoreDefinition - defineAsID: #id; + define: #id as: String; define: #text as: String; define: #created as: String; define: #authorId as: String; define: #timelines as: (Dictionary of: String -> String); define: #metrics as: (Dictionary of: String -> String); + define: #metadata as: (Dictionary of: String -> String); yourself \ No newline at end of file diff --git a/Socialmetrica.package/Tweet.class/instance/fromNitterHtmlItem..st b/Socialmetrica.package/Tweet.class/instance/fromNitterHtmlItem..st index 46273c1..1a5cdbf 100644 --- a/Socialmetrica.package/Tweet.class/instance/fromNitterHtmlItem..st +++ b/Socialmetrica.package/Tweet.class/instance/fromNitterHtmlItem..st @@ -13,8 +13,8 @@ fromNitterHtmlItem: xmlItem copyReplaceAll: 'UTC' with: '+00:00') asDateAndTime. created := timeTemp. - text := (xmlItem xpath: '//div[@class="tweet-content media-body"]') - stringValue. + text := ((xmlItem xpath: '//div[@class="tweet-content media-body"]') asString allButLast) + copyReplaceAll: 'a XPathNodeSet(' with: '' . id := authorAndId last copyReplaceAll: '#m"' with: ''. authorId := self user id. self metricsFromNitterHtml: xmlItem \ No newline at end of file diff --git a/Socialmetrica.package/Tweet.class/instance/metricsFromNitterHtml..st b/Socialmetrica.package/Tweet.class/instance/metricsFromNitterHtml..st index 0eb54bb..242c085 100644 --- a/Socialmetrica.package/Tweet.class/instance/metricsFromNitterHtml..st +++ b/Socialmetrica.package/Tweet.class/instance/metricsFromNitterHtml..st @@ -4,19 +4,19 @@ metricsFromNitterHtml: xmlItem Most times they store numbers, but sometimes, retweets store a userName" self metrics at: 'comment' - put: (((xmlItem xpath: '//div[@class="icon-container"]') + put: (((xmlItem xpath: '//div[@class="tweet-stats"]//div[@class="icon-container"]') select: [ :item | item asString includesSubstring: 'comment' ]) stringValue trimmed copyReplaceAll: ',' with: ''); at: 'retweet' - put: (((xmlItem xpath: '//div[@class="icon-container"]') + put: (((xmlItem xpath: '//div[@class="tweet-stats"]//div[@class="icon-container"]') select: [ :item | item asString includesSubstring: 'retweet' ]) stringValue trimmed copyReplaceAll: ',' with: ''); at: 'quote' - put: (((xmlItem xpath: '//div[@class="icon-container"]') + put: (((xmlItem xpath: '//div[@class="tweet-stats"]//div[@class="icon-container"]') select: [ :item | item asString includesSubstring: 'quote' ]) stringValue trimmed copyReplaceAll: ',' with: ''); at: 'heart' - put: (((xmlItem xpath: '//div[@class="icon-container"]') + put: (((xmlItem xpath: '//div[@class="tweet-stats"]//div[@class="icon-container"]') select: [ :item | item asString includesSubstring: 'heart' ]) stringValue trimmed copyReplaceAll: ',' with: ''). self metadata