Improving urls for load more tweets and removing unnecessary method. ref 5b169c4ebf

Socialmetrica.package/NitterUser.class/instance/defaultConfig.st

@@ -1,5 +0,0 @@
-accessing
-defaultConfig
-
-	self config: { 'folder' -> (FileLocator userData / 'Socialmetrica' / self userName) } asDictionary.
-	^ self config

Socialmetrica.package/NitterUser.class/instance/numberOfURLsForLoadingTweets..st

@@ -1,18 +1,35 @@
 accessing
 numberOfURLsForLoadingTweets: number
 
-	| collectionURLs count asURLs |
-	collectionURLs := {
-	self userNameLink .
-	(self userNameLink, ((self documentTree xPath: '//a[.="Load more"]') @ 'href') stringValue) .} asOrderedCollection.
+	| collectionURLs count asURLs urlAndTweets |
+	number = 1 ifTrue: [ ^ self ].
 	
-	number <= 2 ifTrue: [ ^ collectionURLs ].
-	count := 2.
-	(number-count) timesRepeat: [ | tempDoc | 
-		tempDoc := XMLHTMLParser parse: (collectionURLs at: count) asUrl retrieveContents.
-	collectionURLs 
-		add: (self userNameLink, 
-			((tempDoc xPath: '//a[.="Load more"]') @ 'href') stringValue).
+	urlAndTweets := OrderedDictionary new.
+	collectionURLs := { self userNameLink .}asOrderedCollection.
+	
+	urlAndTweets 
+			at: 'tweets'
+			put: ((self documentTree xpath: '//div[@class="timeline-item "]')asOrderedCollection collect: [ :xmlElement | xmlElement postCopy]).
+	
+	count := 1.
+	(number-count) timesRepeat: [ | tempDoc docTree urlString | 
+		tempDoc := XMLHTMLParser parse: 
+						(collectionURLs at: count) asUrl retrieveContents.
+			
+		urlString := (self userNameLink, 
+				((tempDoc xPath: '//a[.="Load more"]') @ 'href') stringValue).
+		docTree := XMLHTMLParser parse: urlString asUrl retrieveContents.
+		collectionURLs 
+			add: urlString.
+		
+		urlAndTweets 
+			at: 'tweets-', (urlString splitOn: 'cursor=') second
+			put: ((docTree xpath: '//div[@class="timeline-item "]')asOrderedCollection 
+					collect: [ :xmlElement | xmlElement postCopy ]).
 		count := count+1 ].
+		
 	asURLs := collectionURLs collect: [ :string | string asUrl ].
-	^ asURLs.
+	
+	urlAndTweets at: 'urls' put: asURLs.
+	
+	^ urlAndTweets.

Socialmetrica.package/Tweet.class/instance/fromNitterTimelineItem..st

@@ -0,0 +1,12 @@
+accessing
+fromNitterTimelineItem: xmlItem
+	| author authorAndId |
+	authorAndId := (((xmlItem xpath: '//a[@class="tweet-link"]') 
+					asString splitOn: 'href="' ) second splitOn: '/') reject: [ :i | i isEmpty or: [i = '>)']].
+	author := authorAndId first.
+	user := NitterUser new
+		userName: author .
+	"created := (xmlItem xpath: 'pubDate') stringValue.
+	text := (xmlItem xpath: 'description') stringValue."
+	id := authorAndId last copyReplaceAll: '#m"' with: ''.
+	authorId := self user id.