Improving nitter user getting and parsing tweets.

Socialmetrica.package/NitterUser.class/instance/collectRawTweetsForPages..st

@@ -0,0 +1,16 @@
+accessing
+collectRawTweetsForPages: anInteger
+
+	| pagesDict rawResponse response |
+	pagesDict := self pageURLs: anInteger.
+	rawResponse := OrderedCollection new.
+	pagesDict values do: [ :each | 
+		rawResponse addAll: ((each xpath: '//div[@class="timeline-item "]') asOrderedCollection
+							collect: [ :xmlElement | xmlElement postCopy ])
+		 
+	].
+	response := TweetsCollection new.
+	response query: pagesDict keys.
+	response tweets: (rawResponse collect: [ :tweet | Tweet new fromNitterHtmlItem: tweet ]).
+	^ response.
+	

Socialmetrica.package/NitterUser.class/instance/documentTree.st

@@ -1,3 +1,3 @@
 operation
 documentTree
-	^ XMLHTMLParser parse: self userNameLink asUrl retrieveContents
+	^ self documentTreeFor: self userNameLink

Socialmetrica.package/NitterUser.class/instance/documentTreeFor..st

@@ -0,0 +1,3 @@
+accessing
+documentTreeFor: anUrl
+	^ XMLHTMLParser parse:anUrl asUrl retrieveContents

Socialmetrica.package/NitterUser.class/instance/numberOfURLsForLoadingTweets..st

@@ -1,37 +0,0 @@
-accessing
-numberOfURLsForLoadingTweets: number
-
-	| collectionURLs count asURLs urlAndTweets |
-	number = 1 ifTrue: [ ^ self ].
-
-	urlAndTweets := OrderedDictionary new.
-	collectionURLs := { self userNameLink } asOrderedCollection.
-
-	urlAndTweets at: 'tweets' put: self lastTweetsFromHtml.
-
-	count := 1.
-	number - count timesRepeat: [ 
-		| tempDoc docTree urlString |
-		tempDoc := XMLHTMLParser parse:
-			           (collectionURLs at: count) asUrl retrieveContents.
-
-		urlString := self userNameLink
-		             ,
-		             ((tempDoc xPath: '//a[.="Load more"]') @ 'href')
-			             stringValue.
-		docTree := XMLHTMLParser parse: urlString asUrl retrieveContents.
-		collectionURLs add: urlString.
-
-		urlAndTweets
-			at: 'tweets-' , (urlString splitOn: 'cursor=') second
-			put:
-				((docTree xpath: '//div[@class="timeline-item "]')
-					 asOrderedCollection collect: [ :xmlElement | 
-					 xmlElement postCopy ]).
-		count := count + 1 ].
-
-	asURLs := collectionURLs collect: [ :string | string asUrl ].
-
-	urlAndTweets at: 'urls' put: asURLs.
-
-	^ urlAndTweets

Socialmetrica.package/NitterUser.class/instance/pageCursorFor..st

@@ -0,0 +1,10 @@
+accessing
+pageCursorFor: anUrl
+
+	| response value key |
+	response := Dictionary new.
+	value := self documentTreeFor: anUrl.
+	key := ((value xpath: '//a[.="Load more"]') @ 'href')stringValue.
+	^ response 
+		at: key put: value;
+		yourself

Socialmetrica.package/NitterUser.class/instance/pageDocTrees..st

@@ -0,0 +1,14 @@
+accessing
+pageDocTrees: anInteger
+
+	| response nextPageLink previousPageLink |
+	
+	response := OrderedDictionary new.
+	previousPageLink := self userNameLink.
+	response add: previousPageLink.
+	anInteger - 1 timesRepeat: [ 
+		nextPageLink := self userNameLink, (self pageCursorFor:previousPageLink) value.
+		response add: nextPageLink.			
+		previousPageLink := nextPageLink 
+	].
+	^ response

Socialmetrica.package/NitterUser.class/instance/pageURLs..st

@@ -0,0 +1,14 @@
+accessing
+pageURLs: anInteger
+
+	| response nextPageLink previousPageLink dicTemp |
+	
+	response := OrderedDictionary new.
+	response at: self userNameLink put: self documentTree.
+	previousPageLink := self userNameLink.
+	anInteger - 1 timesRepeat: [ 
+		nextPageLink := self userNameLink, (self pageCursorFor:previousPageLink) keys first.
+		response at: nextPageLink put: (self pageCursorFor:previousPageLink) values first.			
+		previousPageLink := nextPageLink 
+	].
+	^ response

Socialmetrica.package/Tweet.class/instance/metricsFromNitterHtml..st

@@ -25,6 +25,6 @@ metricsFromNitterHtml: xmlItem
 				 copyReplaceAll: ','
 				 with: '').
 				 
-	metadata
+	self metadata
 		at: 'pinned'
 		put: (xmlItem xpath: '//div[@class="pinned"]') stringValue trimmed = 'Pinned Tweet'

Socialmetrica.package/TweetsCollection.class/instance/store.st

@@ -1,4 +1,8 @@
 accessing
 store
 	ReStore isConnected ifFalse: [ self class storeDB]. "Starting  the ReStore singleton."
-	self tweets do: [:each | ReStore evaluateAsTransaction: [  each store ] ].
+	self tweets do: [:each | 	ReStore evaluateAsTransaction: [  
+			each store.
+			"each user id isInDB? ifFalse: [ each user store ]" 
+		] 
+	].