Compare commits
No commits in common. "master" and "detached/2" have entirely different histories.
master
...
detached/2
@ -5,13 +5,10 @@ baseline: spec
|
||||
for: #common
|
||||
do: [
|
||||
"Dependencies"
|
||||
"self xmlParserHTML: spec."
|
||||
self reStore: spec.
|
||||
self miniDocs: spec.
|
||||
"self roassal3Exporters: spec."
|
||||
"self tiddlyWikiPharo: spec."
|
||||
self xmlParserHTML: spec.
|
||||
"self rssTools: spec."
|
||||
"Packages"
|
||||
spec
|
||||
package: 'Socialmetrica'
|
||||
with: [ spec requires: #('ReStore' 'MiniDocs' "'XMLParserHTML' 'Roassal3Exporters' 'TiddlyWikiPharo'") ]
|
||||
with: [ spec requires: #('XMLParserHTML' "'RSSTools'") ]
|
||||
]
|
||||
@ -1,8 +0,0 @@
|
||||
accessing
|
||||
miniDocs: spec
|
||||
|
||||
| repo |
|
||||
repo := ExoRepo new
|
||||
repository: 'https://code.sustrato.red/Offray/MiniDocs'.
|
||||
repo load.
|
||||
spec baseline: 'MiniDocs' with: [ spec repository: 'gitlocal://', repo local fullName ]
|
||||
@ -1,10 +0,0 @@
|
||||
baselines
|
||||
reStore: spec
|
||||
|
||||
Metacello new
|
||||
repository: 'github://rko281/ReStoreForPharo';
|
||||
baseline: 'ReStore';
|
||||
onConflict: [ :ex | ex useLoaded ];
|
||||
onWarningLog;
|
||||
load: 'all'.
|
||||
spec baseline: 'ReStore' with: [ spec repository: 'github://rko281/ReStoreForPharo']
|
||||
@ -1,7 +0,0 @@
|
||||
baselines
|
||||
roassal3Exporters: spec
|
||||
Metacello new
|
||||
baseline: 'Roassal3Exporters';
|
||||
repository: 'github://ObjectProfile/Roassal3Exporters';
|
||||
load.
|
||||
spec baseline: 'Roassal3Exporters' with: [ spec repository: 'github://ObjectProfile/Roassal3Exporters']
|
||||
@ -0,0 +1,10 @@
|
||||
baselines
|
||||
rssTools: spec
|
||||
Metacello new
|
||||
repository: 'github://brackendev/RSSTools-Pharo:v1.0.1/src';
|
||||
baseline: 'RSSTools';
|
||||
onConflict: [ :ex | ex useLoaded ];
|
||||
onUpgrade: [ :ex | ex useLoaded ];
|
||||
onDowngrade: [ :ex | ex useLoaded ];
|
||||
load.
|
||||
spec baseline: 'RSSTools' with: [ spec repository: 'github://brackendev/RSSTools-Pharo:v1.0.1/src']
|
||||
@ -1,8 +0,0 @@
|
||||
baselines
|
||||
tiddlyWikiPharo: spec
|
||||
|
||||
| repo |
|
||||
repo := ExoRepo new
|
||||
repository: 'https://code.sustrato.red/Offray/TiddlyWikiPharo'.
|
||||
repo load.
|
||||
spec baseline: 'TiddlyWikiPharo' with: [ spec repository: 'gitlocal://', repo local fullName ]
|
||||
@ -4,6 +4,8 @@ xmlParserHTML: spec
|
||||
baseline: 'XMLParserHTML';
|
||||
repository: 'github://pharo-contributions/XML-XMLParserHTML/src';
|
||||
onConflict: [ :ex | ex useLoaded ];
|
||||
onUpgrade: [ :ex | ex useLoaded ];
|
||||
onDowngrade: [ :ex | ex useLoaded ];
|
||||
onWarningLog;
|
||||
load.
|
||||
spec baseline: 'XMLParserHTML' with: [spec repository: 'github://pharo-contributions/XML-XMLParserHTML/src']
|
||||
@ -6,7 +6,7 @@ To install, first install [ExoRepo](https://code.tupale.co/Offray/ExoRepo) and t
|
||||
|
||||
```smalltalk
|
||||
ExoRepo new
|
||||
repository: 'https://code.sustrato.red/Offray/Socialmetrica';
|
||||
repository: 'https://code.tupale.co/Offray/Socialmetrica';
|
||||
load.
|
||||
```
|
||||
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
accessing
|
||||
columnsDictionary
|
||||
^ {
|
||||
'url' -> 'url' .
|
||||
'healthy' -> 'healthy'.
|
||||
'healthy_percentage_overall' -> 'uptime' .
|
||||
'rss' -> 'rss' .
|
||||
'version' -> 'version'} asDictionary
|
||||
@ -1,4 +0,0 @@
|
||||
accessing
|
||||
exportInstanceReport
|
||||
MiniDocs exportAsSton: self instances on: FileLocator temp / 'instances.ston'.
|
||||
^ FileLocator temp / 'instances.ston'
|
||||
@ -1,8 +0,0 @@
|
||||
accessing
|
||||
instanceRows
|
||||
^ (self instances at: 'hosts') collect: [:rawRow | | newRow |
|
||||
newRow := NitterInstance new.
|
||||
self columnsDictionary keysAndValuesDo: [:key :value |
|
||||
newRow writeSlotNamed: value value: (rawRow at: key) ].
|
||||
newRow
|
||||
].
|
||||
@ -1,3 +0,0 @@
|
||||
accessing
|
||||
instances
|
||||
^ self instanceRows
|
||||
@ -1,9 +0,0 @@
|
||||
accessing
|
||||
instancesCache
|
||||
| cache cacheFile |
|
||||
cacheFile := FileLocator temp / 'nitter-instances.ston'.
|
||||
cacheFile exists
|
||||
ifFalse: [
|
||||
cache := STON fromString: 'https://status.d420.de/api/v1/instances' asUrl retrieveContents.
|
||||
MarkupFile exportAsFileOn: cacheFile containing: cache ].
|
||||
^ STON fromString: cacheFile contents
|
||||
@ -1,2 +0,0 @@
|
||||
accessing
|
||||
instancesTable
|
||||
@ -1,21 +0,0 @@
|
||||
accessing
|
||||
viewInstancesFor: aView
|
||||
<gtView>
|
||||
| columnedList columnNamesMap |
|
||||
self instances isEmptyOrNil ifTrue: [ ^ aView empty].
|
||||
columnedList := aView columnedList
|
||||
title: 'Instances';
|
||||
items: [ self instanceRows ];
|
||||
priority: 1.
|
||||
columnNamesMap := {
|
||||
'Instance' -> 'url'.
|
||||
'Healthy' -> 'healthy'.
|
||||
'Uptime %' -> 'uptime'.
|
||||
'RSS' -> 'rss'.
|
||||
'Nitter Version' -> 'version'} asOrderedDictionary.
|
||||
columnNamesMap keysAndValuesDo: [:aName :value |
|
||||
columnedList
|
||||
column: aName
|
||||
text: [:instanceRow | (instanceRow readSlotNamed: value) ifNil: [''] ]
|
||||
].
|
||||
^ columnedList
|
||||
@ -1,11 +0,0 @@
|
||||
{
|
||||
"commentStamp" : "",
|
||||
"super" : "Object",
|
||||
"category" : "Socialmetrica",
|
||||
"classinstvars" : [ ],
|
||||
"pools" : [ ],
|
||||
"classvars" : [ ],
|
||||
"instvars" : [ ],
|
||||
"name" : "Nitter",
|
||||
"type" : "normal"
|
||||
}
|
||||
@ -1,7 +0,0 @@
|
||||
I model a Nitter instance uptime & health tracker as described in https://status.d420.de/about.
|
||||
|
||||
Taken from the official documentation, in the previous linkm the fields we are modelling are:
|
||||
|
||||
- healthy: stands for hosts which are reachable and pass a content check.
|
||||
- rss: whether the host has RSS feeds enabled.
|
||||
- version: which nitter version the host reports.
|
||||
@ -1,3 +0,0 @@
|
||||
accessing
|
||||
hasRSS
|
||||
^ self rss
|
||||
@ -1,3 +0,0 @@
|
||||
accessing
|
||||
healthy
|
||||
^ healthy
|
||||
@ -1,3 +0,0 @@
|
||||
accessing
|
||||
isHealthy
|
||||
^ self healthy
|
||||
@ -1,5 +0,0 @@
|
||||
accessing
|
||||
printOn: aStream
|
||||
super printOn: aStream.
|
||||
^ aStream
|
||||
nextPutAll: '( ', self url ,' | uptime: ', self uptime asString, ' )'
|
||||
@ -1,3 +0,0 @@
|
||||
accessing
|
||||
rss
|
||||
^ rss
|
||||
@ -1,3 +0,0 @@
|
||||
accessing
|
||||
uptime
|
||||
^ uptime
|
||||
@ -1,3 +0,0 @@
|
||||
accessing
|
||||
url
|
||||
^ url
|
||||
@ -1,17 +0,0 @@
|
||||
{
|
||||
"commentStamp" : "<historical>",
|
||||
"super" : "Object",
|
||||
"category" : "Socialmetrica",
|
||||
"classinstvars" : [ ],
|
||||
"pools" : [ ],
|
||||
"classvars" : [ ],
|
||||
"instvars" : [
|
||||
"url",
|
||||
"healthy",
|
||||
"uptime",
|
||||
"rss",
|
||||
"version"
|
||||
],
|
||||
"name" : "NitterInstance",
|
||||
"type" : "normal"
|
||||
}
|
||||
@ -3,4 +3,4 @@ nitterProvider
|
||||
"For a full list of Nitter providers, see:
|
||||
|
||||
https://github.com/zedeus/nitter/wiki/Instances"
|
||||
^ 'https://nitter.net/'
|
||||
^ 'https://nitter.42l.fr/'
|
||||
@ -1,29 +1,7 @@
|
||||
accessing
|
||||
asDictionary
|
||||
|
||||
| tweets tweetsHistogramData repliesHistogramData quotesHistogramData retweetsHistogramData |
|
||||
[self config at: 'lang']
|
||||
onErrorDo: [ ^ self inform: 'Please put a lang key with a language (for example: en) in the object config.' ].
|
||||
tweets := self messages.
|
||||
tweetsHistogramData := self tweetsByWeeksTimeSpan.
|
||||
repliesHistogramData := self repliesByWeeksTimeSpan.
|
||||
quotesHistogramData := self quotesReportData.
|
||||
retweetsHistogramData := self retweetsReportData.
|
||||
^ { 'profile-card-avatar' -> self profileImageFile fullName.
|
||||
'profile-card-avatar-url' -> self profileImageUrl.
|
||||
^ { 'profile-card-avatar' -> self profileImageFile values first fullName.
|
||||
'profile-card-fullname' -> self name .
|
||||
'profile-card-username' -> self userName .
|
||||
'profile-bio' -> self profileBio.
|
||||
'messages-size' -> tweets size.
|
||||
'messages-newest' -> tweets newest created asDate greaseString.
|
||||
'messages-oldest' -> tweets oldest created asDate greaseString.
|
||||
'tweets-histogram-labels' -> tweetsHistogramData third.
|
||||
'tweets-histogram-quantity' -> tweetsHistogramData second.
|
||||
'replies-histogram-labels' -> repliesHistogramData third.
|
||||
'replies-histogram-quantity' -> repliesHistogramData second.
|
||||
'retweets-histogram-labels' -> retweetsHistogramData third.
|
||||
'retweets-histogram-quantity' -> retweetsHistogramData second.
|
||||
'quotes-histogram-labels' -> quotesHistogramData third.
|
||||
'quotes-histogram-quantity' -> quotesHistogramData second.
|
||||
'wordcloud-data' -> (self wordcloudDataLanguage: (self config at: 'lang')) first.
|
||||
} asDictionary
|
||||
'profile-bio' -> self profileBio } asDictionary
|
||||
@ -1,28 +0,0 @@
|
||||
accessing
|
||||
asDictionaryForWeb
|
||||
|
||||
| tweets tweetsHistogramData repliesHistogramData quotesHistogramData retweetsHistogramData |
|
||||
tweets := self messages.
|
||||
tweetsHistogramData := self tweetsByTimeSpan: 7.
|
||||
repliesHistogramData := self repliesByTimeSpan: 7.
|
||||
quotesHistogramData := self quotesReportData.
|
||||
retweetsHistogramData := self retweetsReportData.
|
||||
|
||||
^ { 'profile-card-avatar' -> self profileImageFile fullName.
|
||||
'profile-card-avatar-url' -> self profileImageUrl.
|
||||
'profile-card-fullname' -> self name .
|
||||
'profile-card-username' -> self userName .
|
||||
'profile-bio' -> self profileBio.
|
||||
'messages-size' -> tweets size.
|
||||
'messages-newest' -> tweets newest created asDate greaseString.
|
||||
'messages-oldest' -> tweets oldest created asDate greaseString.
|
||||
'tweets-histogram-labels' -> tweetsHistogramData third.
|
||||
'tweets-histogram-quantity' -> tweetsHistogramData second.
|
||||
'replies-histogram-labels' -> repliesHistogramData third.
|
||||
'replies-histogram-quantity' -> repliesHistogramData second.
|
||||
'retweets-histogram-labels' -> retweetsHistogramData third.
|
||||
'retweets-histogram-quantity' -> retweetsHistogramData second.
|
||||
'quotes-histogram-labels' -> quotesHistogramData third.
|
||||
'quotes-histogram-quantity' -> quotesHistogramData second.
|
||||
'wordcloud-data' -> self wordcloudData first.
|
||||
} asDictionary
|
||||
@ -1,16 +0,0 @@
|
||||
accessing
|
||||
asTiddler
|
||||
|
||||
| tempDict tiddler |
|
||||
tiddler := Tiddler new.
|
||||
tiddler customFields.
|
||||
tempDict := self asDictionary.
|
||||
tempDict keysAndValuesDo: [ :key :value |
|
||||
tiddler customFields at: key put: value
|
||||
].
|
||||
^ tiddler
|
||||
created;
|
||||
title: (tiddler customFields at: 'profile-card-username');
|
||||
type: 'text/vnd.tiddlywiki';
|
||||
text:
|
||||
'<<image-card "', self profileImageUrl, '" title:', ($' asString), '<$transclude field="profile-card-fullname"/>', ($' asString), 'text:', ($' asString), '<$transclude field="profile-bio"/>', ($' asString), 'footer:', ($' asString), '@<$transclude field="profile-card-username"/>', ($' asString), 'align:"center" pos:"top">>'.
|
||||
@ -1,4 +0,0 @@
|
||||
accessing
|
||||
authorIds
|
||||
|
||||
^ TwitterUser storedInstances select: [ :each | each userName = self userName ] thenCollect: [ :each | each id ]
|
||||
@ -1,12 +1,10 @@
|
||||
accessing
|
||||
avatarPicture
|
||||
| response profileImgFile|
|
||||
profileImgFile := self profileImageFile.
|
||||
profileImgFile exists
|
||||
ifTrue: [ ^ (ImageReadWriter formFromFileNamed: profileImgFile fullName) asElement ].
|
||||
| response tempImage |
|
||||
response := ZnClient new url: (self profileImageUrl); get; response.
|
||||
response contentType = ZnMimeType imageJpeg
|
||||
ifTrue: [ ^ (PluginBasedJPEGReadWriter gtFromBuffer: response contents) asElement ].
|
||||
response contentType = ZnMimeType imagePng
|
||||
ifTrue: [ ^ (PNGReadWriter gtFromBuffer: response contents) asElement ].
|
||||
^ GtABContact new avatar
|
||||
tempImage on: Error do: [ ^ GtABContact new avatar ].
|
||||
^ tempImage value asElement .
|
||||
@ -1,33 +0,0 @@
|
||||
accessing
|
||||
collectRawTweetsFrom: anUrl upToPage: anInteger
|
||||
|
||||
| pagesDict response customQuery |
|
||||
pagesDict := self getPagesContentsFrom: anUrl upTo: anInteger.
|
||||
response := TweetsCollection new.
|
||||
customQuery := Dictionary new
|
||||
at: 'parameters' put: pagesDict keys;
|
||||
at: 'date' put: DateAndTime now;
|
||||
yourself.
|
||||
response query: customQuery.
|
||||
pagesDict keysAndValuesDo: [ :key :rawTweets | | temp |
|
||||
temp := (rawTweets xpath: '//div[@class="timeline-item "]') asOrderedCollection
|
||||
collect: [ :xmlElement | xmlElement postCopy ].
|
||||
temp do: [ :tweet | | tempTweet |
|
||||
tempTweet := Tweet new fromNitterHtmlItem: tweet.
|
||||
tempTweet metadata
|
||||
at: DateAndTime now asString put: key;
|
||||
yourself.
|
||||
response add: tempTweet.
|
||||
]
|
||||
].
|
||||
response messages: (response messages select: [ :tweet | tweet isNotNil ]).
|
||||
response messages doWithIndex: [ :tweet :i |
|
||||
| current previous |
|
||||
current := response messages at: i.
|
||||
i < response lastIndex ifTrue: [
|
||||
previous := response messages at: i + 1.
|
||||
current timelines
|
||||
at: self userName put: previous id;
|
||||
yourself ]].
|
||||
^ response.
|
||||
|
||||
@ -1,5 +0,0 @@
|
||||
accessing
|
||||
collectRawTweetsFromOldestUpToPage: anInteger
|
||||
|
||||
^ self collectRawTweetsFrom: self oldestTweetPageCursor upToPage: anInteger
|
||||
|
||||
@ -1,5 +1,32 @@
|
||||
accessing
|
||||
collectRawTweetsUpToPage: anInteger
|
||||
|
||||
^ self collectRawTweetsFrom: self userNameLinkWithReplies upToPage: anInteger
|
||||
| pagesDict response customQuery |
|
||||
pagesDict := self getPagesContentsUpto: anInteger.
|
||||
response := TweetsCollection new.
|
||||
customQuery := Dictionary new
|
||||
at: 'parameters' put: pagesDict keys;
|
||||
at: 'date' put: DateAndTime now;
|
||||
yourself.
|
||||
response query: customQuery.
|
||||
pagesDict keysAndValuesDo: [ :key :rawTweets | | temp |
|
||||
temp := (rawTweets xpath: '//div[@class="timeline-item "]') asOrderedCollection
|
||||
collect: [ :xmlElement | xmlElement postCopy ].
|
||||
temp do: [ :tweet | | tempTweet |
|
||||
tempTweet := Tweet new fromNitterHtmlItem: tweet.
|
||||
tempTweet metadata
|
||||
at: DateAndTime now asString put: key;
|
||||
yourself.
|
||||
response messages add: tempTweet.
|
||||
]
|
||||
].
|
||||
response messages doWithIndex: [ :tweet :i |
|
||||
| current previous |
|
||||
current := response messages at: i.
|
||||
i < response messages size ifTrue: [
|
||||
previous := response messages at: i + 1.
|
||||
current timelines
|
||||
at: self userName put: previous id;
|
||||
yourself ]].
|
||||
^ response.
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
accessing
|
||||
configureDefaultReportingPeriod
|
||||
[ config at: 'reportingPeriod' ]
|
||||
onErrorDo: [ self config
|
||||
at: 'reportingPeriod'
|
||||
put: (Timespan
|
||||
starting: messages oldest created asDateAndTime
|
||||
ending: messages newest created asDateAndTime + 1 minute) ]
|
||||
@ -2,7 +2,5 @@ accessing
|
||||
createdAt
|
||||
^ createdAt ifNil: [| joinDateString |
|
||||
joinDateString := ((self documentTree xpath: '//div[@class="profile-joindate"]/span/@title') stringValue).
|
||||
|
||||
createdAt := (ZTimestampFormat fromString:'4:05 PM - 3 Feb 2001') parse: joinDateString.
|
||||
createdAt := createdAt asDateAndTime
|
||||
createdAt := (ZTimestampFormat fromString:'4:05 PM - 03 Feb 2001') parse: joinDateString.
|
||||
]
|
||||
@ -1,3 +1,3 @@
|
||||
operation
|
||||
documentTree
|
||||
^ self documentTreeFor: self userNameLinkWithReplies
|
||||
^ self documentTreeFor: (self userNameLink, '/with_replies')
|
||||
@ -1,12 +1,4 @@
|
||||
accessing
|
||||
downloadProfileImage
|
||||
|
||||
self remoteIsFound
|
||||
ifTrue: [ ^ self exportProfileImageOn: self folder / 'profile-image', 'jpg' ]
|
||||
ifFalse: [ | tempFile |
|
||||
tempFile := (self folder / 'profile-image', 'jpg') asFileReference.
|
||||
tempFile ensureCreateFile.
|
||||
tempFile binaryWriteStreamDo: [ :stream |
|
||||
stream nextPutAll: 'https://mutabit.com/repos.fossil/mutabit/uv/wiki/commons/twitter-user-image-default.jpg' asUrl retrieveContents.
|
||||
super class inform: 'Exported as: ', String cr, tempFile fullName.
|
||||
^ self folder / 'profile-image', 'jpg' ]]
|
||||
^ self exportProfileImageOn: self folder / 'profile-image', 'jpg'
|
||||
@ -1,40 +0,0 @@
|
||||
as yet unclassified
|
||||
downloadWithRenaming: fileReference
|
||||
|
||||
| file tempFile fileHash tempFileHash |
|
||||
file := fileReference asFileReference .
|
||||
tempFile := FileLocator temp / fileReference basename.
|
||||
tempFile ensureCreateFile.
|
||||
tempFile binaryWriteStreamDo: [ :stream |
|
||||
stream nextPutAll: profileImageUrl asUrl retrieveContents.
|
||||
super class inform: 'Exported as: ', String cr, tempFile fullName.
|
||||
].
|
||||
|
||||
OSSUnixSubprocess new
|
||||
command: 'openssl';
|
||||
arguments: { 'dgst' . '-sha256' . file fullName};
|
||||
workingDirectory: (self folder) fullName;
|
||||
redirectStdout;
|
||||
redirectStderr;
|
||||
runAndWaitOnExitDo: [ :process :outString |
|
||||
fileHash := (outString splitOn: ' ') second trimmed].
|
||||
|
||||
OSSUnixSubprocess new
|
||||
command: 'openssl';
|
||||
arguments: { 'dgst' . '-sha256' . tempFile fullName};
|
||||
workingDirectory: (self folder) fullName;
|
||||
redirectStdout;
|
||||
redirectStderr;
|
||||
runAndWaitOnExitDo: [ :process :outString |
|
||||
tempFileHash := (outString splitOn: ' ' ) second trimmed].
|
||||
|
||||
fileHash = tempFileHash
|
||||
ifFalse: [
|
||||
file copyTo: self folder /
|
||||
(file basenameWithoutExtension , '-',
|
||||
('-' join:
|
||||
((file creationTime asLocalStringYMDHM) splitOn: ' ')), '.jpg').
|
||||
file ensureDelete.
|
||||
^ { 'Profile image changed' ->
|
||||
(tempFile moveTo: file)} asDictionary ].
|
||||
^ { 'Same Profile Image' -> file } asDictionary
|
||||
@ -1,6 +1,4 @@
|
||||
accessing
|
||||
exportDefaultReport
|
||||
|
||||
(self hasFolder: 'templates')
|
||||
ifFalse: [ self installTemplate ].
|
||||
^ self exportWithTemplate: (TweetsCollection dataStore / 'templates' / 'template.mus.tex') into: self folder
|
||||
^ self exportWithTemplate: TweetsCollection dataStore / 'templates' / 'template.mus.tex' into: self folder
|
||||
@ -1,9 +0,0 @@
|
||||
accessing
|
||||
exportEmptyHistogramNamed: aDictionary
|
||||
|
||||
| histogram |
|
||||
histogram := RSChart new.
|
||||
histogram extent: (aDictionary at: 'extent').
|
||||
histogram build.
|
||||
histogram canvas exportAsFileNamed: (aDictionary at: 'messagesType'), '-histogram' into: self folder.
|
||||
^ self
|
||||
@ -1,49 +0,0 @@
|
||||
accessing
|
||||
exportHistogramFor: aDictionary By: aTypeString
|
||||
"TODO: quotes and retweets"
|
||||
| messagesDict histogram diagram tempMessages labels subtotals |
|
||||
tempMessages := self perform: (aDictionary at: 'messagesType') asSymbol.
|
||||
|
||||
tempMessages ifEmpty: [ self exportEmptyHistogramNamed:
|
||||
(aDictionary at: 'messagesType'), '-histogram' ].
|
||||
|
||||
((((aDictionary at: 'messagesType') = 'tweets') or: [(aDictionary at: 'messagesType') = 'replies'])
|
||||
and: aTypeString isNumber)
|
||||
ifTrue: [ messagesDict := tempMessages splitBytimeSpansOf: aTypeString ].
|
||||
(aTypeString = 'day' or: [ aTypeString = 'days' ])
|
||||
ifTrue: [ messagesDict := tempMessages splitByDays ].
|
||||
(aTypeString = 'week' or: [ aTypeString = 'weeks' ])
|
||||
ifTrue: [ messagesDict := tempMessages splitByWeeks ].
|
||||
|
||||
(((aDictionary at: 'messagesType') = 'retweets') or: [ (aDictionary at: 'messagesType') = 'quotes' ])
|
||||
ifTrue: [
|
||||
messagesDict := tempMessages asMessagesUserNamesSortedByOccurrences.
|
||||
((aTypeString > messagesDict size) or: [ aTypeString = 1 ])
|
||||
ifFalse: [ | keysToRemove |
|
||||
keysToRemove := OrderedCollection new.
|
||||
1 to: messagesDict size - aTypeString do:
|
||||
[ :i | keysToRemove add: (messagesDict keys at: i + aTypeString) ].
|
||||
messagesDict removeKeys: keysToRemove. ].
|
||||
labels := messagesDict keys.
|
||||
labels := labels collect: [ :profiles | ('@', profiles) ].
|
||||
subtotals := messagesDict values
|
||||
]
|
||||
ifFalse: [ labels := messagesDict keys.
|
||||
subtotals := (messagesDict values collect: [ :collection | collection size ])].
|
||||
|
||||
histogram := RSChart new.
|
||||
histogram extent: (aDictionary at: 'extent').
|
||||
diagram := RSBarPlot new
|
||||
y: subtotals.
|
||||
diagram color: (aDictionary at: 'color').
|
||||
histogram addPlot: diagram.
|
||||
histogram addDecoration: (RSHorizontalTick new
|
||||
fromNames: labels;
|
||||
labelRotation: 0;
|
||||
fontSize: 68 /messagesDict size;
|
||||
yourself).
|
||||
histogram addDecoration: (RSVerticalTick new
|
||||
integer;
|
||||
fontSize: 68 /messagesDict size).
|
||||
histogram build.
|
||||
^ histogram canvas exportAsFileNamed: (aDictionary at: 'messagesType'), '-histogram' into: self folder
|
||||
@ -1,11 +0,0 @@
|
||||
accessing
|
||||
exportOverviewReportLatex
|
||||
|
||||
| weeks floor divisions |
|
||||
weeks := ((self newestTweet created - self oldestTweet created) days / 7).
|
||||
floor := weeks floor.
|
||||
(weeks - floor) > 0.4
|
||||
ifTrue: [ divisions := floor ]
|
||||
ifFalse: [ divisions := floor + 1 ].
|
||||
self exportOverviewReportLatexWithBars: divisions.
|
||||
^ self folder
|
||||
@ -1,11 +0,0 @@
|
||||
accessing
|
||||
exportOverviewReportLatexWithBars: anInteger
|
||||
|
||||
self
|
||||
exportDefaultReport;
|
||||
externalWordCloud;
|
||||
exportTweetsHistogramWithBars: anInteger;
|
||||
exportRetweetsHistogramWithBars: anInteger;
|
||||
exportRepliesHistogramWithBars: anInteger;
|
||||
exportQuotesHistogramWithBars: anInteger.
|
||||
^ self folder
|
||||
@ -1,13 +1,48 @@
|
||||
accessing
|
||||
exportProfileImageOn: fileReference
|
||||
|
||||
| file |
|
||||
| file tempFile tempFileHash fileHash |
|
||||
file := fileReference asFileReference.
|
||||
file exists ifFalse: [
|
||||
file ensureCreateFile.
|
||||
file exists
|
||||
ifFalse: [ file ensureCreateFile.
|
||||
file binaryWriteStreamDo: [ :stream |
|
||||
stream nextPutAll: profileImageUrl asUrl retrieveContents ].
|
||||
self class inform: 'Exported as: ', String cr, file fullName.
|
||||
^ file
|
||||
].
|
||||
self downloadWithRenaming: fileReference
|
||||
stream nextPutAll: profileImageUrl retrieveContents ].
|
||||
super class inform: 'Exported as: ', String cr, file fullName.
|
||||
^ file]
|
||||
|
||||
ifTrue: [
|
||||
tempFile := FileLocator temp / fileReference basename.
|
||||
tempFile ensureCreateFile.
|
||||
tempFile binaryWriteStreamDo: [ :stream |
|
||||
stream nextPutAll: profileImageUrl asUrl retrieveContents.
|
||||
super class
|
||||
inform: 'Exported as: ', String cr, tempFile fullName. ].
|
||||
|
||||
OSSUnixSubprocess new
|
||||
command: 'openssl';
|
||||
arguments: { 'dgst' . '-sha256' . file fullName};
|
||||
workingDirectory: (self folder)fullName;
|
||||
redirectStdout;
|
||||
redirectStderr;
|
||||
runAndWaitOnExitDo: [ :process :outString |
|
||||
fileHash := (outString splitOn: ' ' ) second trimmed].
|
||||
|
||||
OSSUnixSubprocess new
|
||||
command: 'openssl';
|
||||
arguments: { 'dgst' . '-sha256' . tempFile fullName};
|
||||
workingDirectory: (self folder)fullName;
|
||||
redirectStdout;
|
||||
redirectStderr;
|
||||
runAndWaitOnExitDo: [ :process :outString |
|
||||
tempFileHash := (outString splitOn: ' ' ) second trimmed].
|
||||
|
||||
fileHash = tempFileHash
|
||||
ifFalse: [
|
||||
file copyTo: self folder /
|
||||
(file basenameWithoutExtension , '-',
|
||||
('-' join:
|
||||
((file creationTime asLocalStringYMDHM) splitOn: ' ')), '.jpg').
|
||||
file ensureDelete.
|
||||
^ { 'Profile image changed' ->
|
||||
(tempFile moveTo: file)} asDictionary ]].
|
||||
^ { 'Same Profile Image' -> file } asDictionary
|
||||
@ -1,49 +0,0 @@
|
||||
accessing
|
||||
exportQuotesHistogramWithBars: aNumberOfBars
|
||||
|
||||
| quotesDict |
|
||||
quotesDict := {
|
||||
'messagesType' -> 'quotes'.
|
||||
'extent' -> (800@200).
|
||||
'color' -> (Color r:(89/255) g:(217/255) b:(95/255))
|
||||
} asDictionary.
|
||||
^ self exportHistogramFor: quotesDict By: aNumberOfBars
|
||||
|
||||
|
||||
"| keysToRemove quotes labels quotesHistogram diagram |
|
||||
quotes := self quotes asMessagesUserNamesSortedByOccurrences.
|
||||
(aNumberOfBars > quotes size) ifTrue: [ ^ self exportQuotesHistogram ].
|
||||
keysToRemove := OrderedCollection new.
|
||||
1 to: quotes size - aNumberOfBars do:
|
||||
[ :i | keysToRemove add: (quotes keys at: i + aNumberOfBars) ].
|
||||
quotes removeKeys: keysToRemove.
|
||||
|
||||
labels := quotes keys.
|
||||
labels := labels collect: [ :profiles | ('@', profiles) ].
|
||||
quotesHistogram := RSChart new.
|
||||
quotesHistogram extent: 800@200.
|
||||
diagram := RSBarPlot new
|
||||
y: quotes values.
|
||||
diagram color: (Color r:(89/255) g:(217/255) b:(95/255)).
|
||||
quotesHistogram addPlot: diagram.
|
||||
quotesHistogram addDecoration: (RSHorizontalTick new
|
||||
fromNames: labels;
|
||||
labelRotation: 0;
|
||||
fontSize: 72 /quotes size;
|
||||
yourself).
|
||||
quotesHistogram addDecoration: (RSVerticalTick new
|
||||
asFloat: 2;
|
||||
fontSize: 72 /quotes size).
|
||||
quotesHistogram build.
|
||||
|
||||
quotesHistogram canvas pdfExporter
|
||||
zoomToShapes;
|
||||
noFixedShapes;
|
||||
fileName: (self folder / 'quotes-histogram.pdf')fullName;
|
||||
export.
|
||||
quotesHistogram canvas pngExporter
|
||||
zoomToShapes;
|
||||
noFixedShapes;
|
||||
fileName: (self folder / 'quotes-histogram.png')fullName;
|
||||
export.
|
||||
^ self folder / 'quotes-histogram.png'"
|
||||
@ -1,10 +1,34 @@
|
||||
accessing
|
||||
exportRepliesHistogramWithBars: aNumberOfBars
|
||||
|
||||
| repliesDict |
|
||||
repliesDict := {
|
||||
'messagesType' -> 'replies'.
|
||||
'extent' -> (800@200).
|
||||
'color' -> (Color r:(246/255) g:(185/255) b:(46/255))
|
||||
} asDictionary.
|
||||
^ self exportHistogramFor: repliesDict By: aNumberOfBars
|
||||
| tweetsByTimeSpan subtotals repliesColor x tweetsHistogram diagram |
|
||||
|
||||
tweetsByTimeSpan := self collectMessages: [ self replies ] byTimeSpanSplits: aNumberOfBars.
|
||||
subtotals := tweetsByTimeSpan values collect: [ :collection | collection size ].
|
||||
repliesColor := (Color r:(246/255) g:(185/255) b:(46/255)).
|
||||
x := 1 to: subtotals size.
|
||||
tweetsHistogram := RSChart new.
|
||||
tweetsHistogram extent: 800@200.
|
||||
diagram := RSBarPlot new x: x y:subtotals.
|
||||
diagram color: repliesColor.
|
||||
tweetsHistogram addPlot: diagram.
|
||||
tweetsHistogram addDecoration: (RSHorizontalTick new
|
||||
fromNames: tweetsByTimeSpan keys;
|
||||
labelRotation: 0;
|
||||
fontSize: 80 /aNumberOfBars;
|
||||
yourself).
|
||||
tweetsHistogram addDecoration: (RSVerticalTick new
|
||||
integer;
|
||||
fontSize: 80 /aNumberOfBars).
|
||||
tweetsHistogram build.
|
||||
tweetsHistogram canvas pngExporter
|
||||
zoomToShapes;
|
||||
noFixedShapes;
|
||||
fileName: (self folder / 'replies-histogram.png')fullName;
|
||||
export.
|
||||
tweetsHistogram canvas pdfExporter
|
||||
zoomToShapes;
|
||||
noFixedShapes;
|
||||
fileName: (self folder / 'replies-histogram.pdf')fullName;
|
||||
export.
|
||||
^ self folder / 'replies-histogram.png'
|
||||
@ -1,10 +0,0 @@
|
||||
accessing
|
||||
exportRetweetsHistogramWithBars: aNumberOfBars
|
||||
|
||||
| retweetsDict |
|
||||
retweetsDict := {
|
||||
'messagesType' -> 'retweets'.
|
||||
'extent' -> (800@200).
|
||||
'color' -> (Color r:(217/255) g:(56/255) b: (124/255))
|
||||
} asDictionary.
|
||||
^ self exportHistogramFor: retweetsDict By: aNumberOfBars
|
||||
@ -1,8 +0,0 @@
|
||||
accessing
|
||||
exportStaticWebReport
|
||||
|
||||
(self hasFolder: 'commons')
|
||||
ifFalse: [ self installCommons ].
|
||||
(self hasFolder: 'templates')
|
||||
ifFalse: [ self installTemplate ].
|
||||
^ self exportWithTemplate: (TweetsCollection dataStore / 'templates' / 'index.mus.html') into: self folder
|
||||
@ -1,10 +1,34 @@
|
||||
accessing
|
||||
exportTweetsHistogramWithBars: aNumberOfBars
|
||||
|
||||
| tweetsDict |
|
||||
tweetsDict := {
|
||||
'messagesType' -> 'tweets'.
|
||||
'extent' -> (800@200).
|
||||
'color' -> (Color r:(91/255) g:(131/255) b:(222/255))
|
||||
} asDictionary.
|
||||
^ self exportHistogramFor: tweetsDict By: aNumberOfBars
|
||||
| tweetsByTimeSpan subtotals tweetsColor x tweetsHistogram diagram |
|
||||
|
||||
tweetsByTimeSpan := self collectMessages: [ self tweets] byTimeSpanSplits: aNumberOfBars.
|
||||
subtotals := tweetsByTimeSpan values collect: [ :collection | collection size ].
|
||||
tweetsColor := (Color r:(91/255) g:(131/255) b:(222/255)).
|
||||
x := 1 to: subtotals size.
|
||||
tweetsHistogram := RSChart new.
|
||||
tweetsHistogram extent: 800@200.
|
||||
diagram := RSBarPlot new x: x y:subtotals.
|
||||
diagram color: tweetsColor.
|
||||
tweetsHistogram addPlot: diagram.
|
||||
tweetsHistogram addDecoration: (RSHorizontalTick new
|
||||
fromNames: tweetsByTimeSpan keys;
|
||||
labelRotation: 0;
|
||||
fontSize: 80 /aNumberOfBars;
|
||||
yourself).
|
||||
tweetsHistogram addDecoration: (RSVerticalTick new
|
||||
integer;
|
||||
fontSize: 80 /aNumberOfBars).
|
||||
tweetsHistogram build.
|
||||
tweetsHistogram canvas pngExporter
|
||||
zoomToShapes;
|
||||
noFixedShapes;
|
||||
fileName: (self folder / 'tweets-histogram.png')fullName;
|
||||
export.
|
||||
tweetsHistogram canvas pdfExporter
|
||||
zoomToShapes;
|
||||
noFixedShapes;
|
||||
fileName: (self folder / 'tweets-histogram.pdf')fullName;
|
||||
export.
|
||||
^ self folder / 'tweets-histogram.png'
|
||||
@ -1,11 +0,0 @@
|
||||
accessing
|
||||
exportWeekReportLatexBeginningAt: aDay
|
||||
|
||||
self config
|
||||
at: 'reportingPeriod'
|
||||
put: (Timespan starting: aDay asDateAndTime ending: ( aDay asDateAndTime + 7 days)).
|
||||
self messages: self messages.
|
||||
self
|
||||
exportDefaultReport;
|
||||
externalWordCloud.
|
||||
^ self folder
|
||||
@ -1,29 +1,13 @@
|
||||
accessing
|
||||
exportWithTemplate: mustacheFile into: folder
|
||||
|
||||
| tempDictionary bioModified userModified nameModified |
|
||||
| tempDictionary modified |
|
||||
tempDictionary := self asDictionary copy.
|
||||
(mustacheFile fullName endsWith: '.html')
|
||||
ifTrue: [ ^ MarkupFile
|
||||
exportAsFileOn: (folder / self userName , 'html')
|
||||
containing:(mustacheFile asMustacheTemplate value: tempDictionary)].
|
||||
|
||||
(mustacheFile fullName endsWith: '.tex')
|
||||
ifTrue: [ bioModified := self asDictionary at: 'profile-bio'.
|
||||
bioModified := bioModified copyReplaceAll: '@' with: '\@'.
|
||||
bioModified := bioModified copyReplaceAll: '_' with: '\_'.
|
||||
bioModified := bioModified copyReplaceAll: '#' with: '\#'.
|
||||
bioModified := bioModified copyReplaceAll: '👑😎' with: ''.
|
||||
bioModified := bioModified copyReplaceAll: '🇨🇴' with: ''.
|
||||
bioModified := bioModified copyReplaceAll: '|' with: ''.
|
||||
userModified := self asDictionary at: 'profile-card-username'.
|
||||
userModified := userModified copyReplaceAll: '_' with: '\_'.
|
||||
nameModified := self asDictionary at: 'profile-card-fullname'.
|
||||
nameModified := nameModified copyReplaceAll: '🇨🇴' with: ''.
|
||||
tempDictionary at: 'profile-bio' put: bioModified;
|
||||
at: 'profile-card-username' put: userModified;
|
||||
at: 'profile-card-fullname' put: nameModified.
|
||||
modified := self asDictionary at: 'profile-bio'.
|
||||
modified := modified copyReplaceAll: '@' with: '\@'.
|
||||
modified := modified copyReplaceAll: '_' with: '\_'.
|
||||
modified := modified copyReplaceAll: '#' with: '\#'.
|
||||
tempDictionary at: 'profile-bio' put: modified.
|
||||
^ MarkupFile
|
||||
exportAsFileOn: (folder / self userName , 'tex')
|
||||
containing:(mustacheFile asMustacheTemplate value: tempDictionary)]
|
||||
|
||||
containing:(mustacheFile asMustacheTemplate value: tempDictionary)
|
||||
@ -1,12 +1,11 @@
|
||||
accessing
|
||||
externalWordCloud
|
||||
"TO DO: refactor with externalWordCloudWithLanguage:"
|
||||
|
||||
| text outputFile |
|
||||
self areCommonFilesInstalled
|
||||
ifFalse: [ self installCommons ].
|
||||
ifFalse: [ self installExternalWordCloudCommons ].
|
||||
self writeWordsFile.
|
||||
outputFile := (self folder / 'wordcloud.png') fullName.
|
||||
outputFile := (self folder / 'nube.png') fullName.
|
||||
text := (self folder / 'words', 'txt') fullName.
|
||||
OSSUnixSubprocess new
|
||||
command: 'wordcloud_cli';
|
||||
@ -18,8 +17,7 @@ externalWordCloud
|
||||
'--background' . 'white' .
|
||||
'--mode' . 'RGBA' .
|
||||
'--stopwords' . '../../../commons/stopwords-es.txt' .
|
||||
"'--mask' . '../../../commons/nube-mascara.jpg'"
|
||||
};
|
||||
'--mask' . '../../../commons/nube-mascara.jpg'};
|
||||
workingDirectory: self folder fullName;
|
||||
redirectStdout;
|
||||
redirectStderr;
|
||||
|
||||
@ -1,26 +0,0 @@
|
||||
accessing
|
||||
externalWordCloudWithLanguage: language
|
||||
"I proces and render a wordcloud image without stopwords in: es and en."
|
||||
|
||||
| text outputFile |
|
||||
self areCommonFilesInstalled
|
||||
ifFalse: [ self installCommons ].
|
||||
self writeWordsFile.
|
||||
outputFile := (self folder / 'wordcloud.png') fullName.
|
||||
text := (self folder / 'words', 'txt') fullName.
|
||||
OSSUnixSubprocess new
|
||||
command: 'wordcloud_cli';
|
||||
arguments: { '--text' . text .
|
||||
'--imagefile' . outputFile .
|
||||
'--color' . '#5B83DE' .
|
||||
'--width' . '1153' .
|
||||
'--height' . '357' .
|
||||
'--background' . 'white' .
|
||||
'--mode' . 'RGBA' .
|
||||
'--stopwords' . '../../../commons/stopwords-', language, '.txt'.
|
||||
"'--mask' . '../../../commons/nube-mascara.jpg'"
|
||||
};
|
||||
workingDirectory: self folder fullName;
|
||||
redirectStdout;
|
||||
redirectStderr;
|
||||
runAndWaitOnExitDo: [ :process :outString | ^ outputFile asFileReference ]
|
||||
@ -1,5 +1,4 @@
|
||||
accessing
|
||||
getMessages
|
||||
|
||||
self getLocalMessages ifNil: [ self getRemoteMessagesFromHtml ].
|
||||
self getLocalMessages ifNil: [ self getRemoteMessagesFromRss ].
|
||||
^ self messages
|
||||
@ -1,19 +0,0 @@
|
||||
accessing
|
||||
getPagesContentsFrom: anURL upTo: anInteger
|
||||
"I retroactively get all pages contents until a specified page number.
|
||||
|
||||
TO DO: should this be splitted back to two methods, one getting the page urls and other its content?
|
||||
or do we always be getting the cursor urls and its contents all the time.
|
||||
[ ] Benchmark alternative approaches."
|
||||
| response nextPageLink previousPageLink |
|
||||
|
||||
response := OrderedDictionary new.
|
||||
response at: anURL put: (self documentTreeFor: anURL).
|
||||
previousPageLink := anURL.
|
||||
anInteger - 1 timesRepeat: [ | pageCursor |
|
||||
pageCursor := self pageCursorFor:previousPageLink.
|
||||
nextPageLink := self userNameLink, '/with_replies', pageCursor keys first.
|
||||
response at: nextPageLink put: (XMLHTMLParser parse:nextPageLink asUrl retrieveContents).
|
||||
previousPageLink := nextPageLink
|
||||
].
|
||||
^ response
|
||||
@ -1,4 +0,0 @@
|
||||
accessing
|
||||
getPagesContentsFromOldestUpto: anInteger
|
||||
|
||||
^ self getPagesContentsFrom: ((self oldestTweet metadata select: [ :item | item isString and: [ item beginsWith: 'https://' ]]) values first) upTo: anInteger
|
||||
@ -5,4 +5,15 @@ getPagesContentsUpto: anInteger
|
||||
TO DO: should this be splitted back to two methods, one getting the page urls and other its content?
|
||||
or do we always be getting the cursor urls and its contents all the time.
|
||||
[ ] Benchmark alternative approaches."
|
||||
^ self getPagesContentsFrom: self userNameLinkWithReplies upTo: anInteger
|
||||
| response nextPageLink previousPageLink |
|
||||
|
||||
response := OrderedDictionary new.
|
||||
response at: (self userNameLink, '/with_replies') put: self documentTree.
|
||||
previousPageLink := (self userNameLink, '/with_replies').
|
||||
anInteger - 1 timesRepeat: [ | pageCursor |
|
||||
pageCursor := self pageCursorFor:previousPageLink.
|
||||
nextPageLink := self userNameLink, '/with_replies', pageCursor keys first.
|
||||
response at: nextPageLink put: pageCursor values first.
|
||||
previousPageLink := nextPageLink
|
||||
].
|
||||
^ response
|
||||
@ -1,4 +0,0 @@
|
||||
accessing
|
||||
getRemoteMessagesFromHtml
|
||||
|
||||
^ messages := self collectRawTweetsUpToPage: 1
|
||||
@ -0,0 +1,22 @@
|
||||
accessing
|
||||
getRemoteMessagesFromRss
|
||||
|
||||
| customQuery lastTweetsRaw lastTweets |
|
||||
lastTweetsRaw := self rssFeed xmlDocument xpath: '//item'.
|
||||
lastTweets := TweetsCollection new.
|
||||
customQuery := Dictionary new
|
||||
at: 'parameters' put: self userNameLink;
|
||||
at: 'date' put: DateAndTime now;
|
||||
yourself.
|
||||
lastTweets query: customQuery.
|
||||
lastTweetsRaw doWithIndex: [ :rssTweet :i |
|
||||
| current previous |
|
||||
current := Tweet new fromNitterRssItem: rssTweet.
|
||||
i < lastTweetsRaw size ifTrue: [
|
||||
previous := Tweet new fromNitterRssItem: (lastTweetsRaw at: i + 1).
|
||||
current timelines
|
||||
at: self userName put: previous id;
|
||||
yourself ].
|
||||
current queries add: customQuery.
|
||||
lastTweets add: current ].
|
||||
self messages: lastTweets
|
||||
@ -1,6 +0,0 @@
|
||||
accessing
|
||||
hasFolder: folderName
|
||||
|
||||
| fullFolderPath |
|
||||
fullFolderPath :=(TweetsCollection dataStore / folderName ).
|
||||
^ fullFolderPath exists and: [ fullFolderPath children isNotEmpty ]
|
||||
@ -1,7 +1,3 @@
|
||||
accessing
|
||||
id
|
||||
^ id ifNil: [
|
||||
self profileImageUrl
|
||||
ifNil: [ id := 0 ]
|
||||
ifNotNil: [ id := (self profileImageUrl asUrl segments select: [ :each | each isAllDigits ]) first. ]
|
||||
]
|
||||
^ id ifNil: [ id := (self profileImageUrl asUrl segments select: [ :each | each isAllDigits ]) first.]
|
||||
@ -1,22 +0,0 @@
|
||||
accessing
|
||||
installCommons
|
||||
|
||||
| commonFiles folder |
|
||||
commonFiles := #(
|
||||
'https://mutabit.com/repos.fossil/mutabit/raw?name=wiki/commons/stopwords-es.txt&ci=tip'
|
||||
'https://mutabit.com/repos.fossil/mutabit/raw?name=wiki/commons/stopwords-en.txt&ci=tip'
|
||||
'https://mutabit.com/repos.fossil/mutabit/uv/wiki/commons/nube-mascara.jpg'
|
||||
'https://mutabit.com/repos.fossil/mutabit/uv/wiki/commons/logo-mutabit-negro.png').
|
||||
folder := TweetsCollection dataStore / 'commons'.
|
||||
folder exists
|
||||
ifTrue: [ folder ensureDeleteAllChildren ]
|
||||
ifFalse: [ folder ensureCreateDirectory].
|
||||
commonFiles do: [ :fileUrl | | temp |
|
||||
ZnClient new
|
||||
url: fileUrl;
|
||||
downloadTo: folder.
|
||||
temp := (folder children select: [ :file | file basename includesSubstring: 'raw' ]).
|
||||
temp isNotEmpty ifTrue: [
|
||||
temp first renameTo: (((fileUrl splitOn: 'raw?') second splitOn: '/') last removeSuffix: '&ci=tip')].
|
||||
].
|
||||
^ folder
|
||||
@ -0,0 +1,19 @@
|
||||
accessing
|
||||
installExternalWordCloudCommons
|
||||
|
||||
| commonFiles folder |
|
||||
commonFiles := #(
|
||||
'https://mutabit.com/repos.fossil/mutabit/uv/wiki/commons/nube-mascara.jpg'
|
||||
'https://mutabit.com/repos.fossil/mutabit/raw?name=wiki/commons/stopwords-es.txt&ci=tip').
|
||||
folder := TweetsCollection dataStore / 'commons'.
|
||||
folder exists
|
||||
ifTrue: [ folder ensureDeleteAllChildren ]
|
||||
ifFalse: [ folder ensureCreateDirectory].
|
||||
commonFiles do: [ :fileUrl |
|
||||
ZnClient new
|
||||
url: fileUrl;
|
||||
downloadTo: folder].
|
||||
(folder children detect: [ :file | file basename includesSubstring: 'raw' ])
|
||||
renameTo: (((commonFiles second splitOn: 'raw?') second splitOn: '/') last removeSuffix: '&ci=tip').
|
||||
^ folder
|
||||
|
||||
@ -3,11 +3,7 @@ installTemplate
|
||||
|
||||
| templateFiles folder |
|
||||
templateFiles := #(
|
||||
'https://mutabit.com/repos.fossil/mutabit/raw?name=plantillas/TwentySecondsCV/twentysecondcvMod.cls&ci=tip'
|
||||
'https://mutabit.com/repos.fossil/mutabit/raw?name=plantillas/TwentySecondsCV/template.mus.tex&ci=tip'
|
||||
'https://mutabit.com/repos.fossil/mutabit/raw?name=plantillas/SarissaPersonalBlog/index.mus.html&ci=tip'
|
||||
'https://mutabit.com/repos.fossil/mutabit/raw?name=plantillas/SarissaPersonalBlog/output.css&ci=tip'
|
||||
'https://mutabit.com/repos.fossil/mutabit/raw?name=plantillas/SarissaPersonalBlog/echarts-wordcloud.min.js&ci=tip').
|
||||
'https://mutabit.com/repos.fossil/mutabit/raw?name=plantillas/TwentySecondsCV/twentysecondcvMod.cls&ci=tip' 'https://mutabit.com/repos.fossil/mutabit/raw?name=plantillas/TwentySecondsCV/template.mus.tex&ci=tip').
|
||||
folder := TweetsCollection dataStore / 'templates'.
|
||||
folder exists
|
||||
ifTrue: [ folder ensureDeleteAllChildren ]
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
accessing
|
||||
messages
|
||||
messages ifNil: [ messages := TweetsCollection new ].
|
||||
messages ifEmpty: [ self getLocalMessages ].
|
||||
messages ifEmpty: [ self getRemoteMessagesFromHtml ].
|
||||
config at: 'reportingPeriod' ifAbsent: [ ^ messages ].
|
||||
"self configureDefaultReportingPeriod."
|
||||
^ messages
|
||||
select: [ :message |
|
||||
message created
|
||||
between: self reportingPeriod start
|
||||
and: self reportingPeriod end ]
|
||||
@ -1,6 +1,3 @@
|
||||
accessing
|
||||
name
|
||||
|
||||
| documentTree |
|
||||
documentTree := [ self documentTree ] onErrorDo: [ ^ nil ].
|
||||
^ name ifNil: [ name := (documentTree xpath: '//div[@class="profile-card-tabs-name"]//a[@class="profile-card-fullname"]') stringValue ]
|
||||
^ name ifNil: [ name := ((self rssFeed requiredItems title) splitOn: '/') first ]
|
||||
@ -1,4 +0,0 @@
|
||||
accessing
|
||||
newestTweet
|
||||
|
||||
^ (self tweets select: [ :tweet | tweet created = ((self tweets collect: [ :each | each created ]) asSortedCollection last)]) first.
|
||||
@ -1,4 +0,0 @@
|
||||
accessing
|
||||
oldestTweet
|
||||
|
||||
^ (self tweets select: [ :tweet | tweet created = ((self tweets collect: [ :each | each created ]) asSortedCollection first)]) first.
|
||||
@ -1,3 +0,0 @@
|
||||
accessing
|
||||
oldestTweetPageCursor
|
||||
^ (self oldestTweet metadata select: [ :item | item isString and: [ item beginsWith: 'https://' ]]) values first value
|
||||
@ -2,8 +2,6 @@ accessing
|
||||
profileImageFile
|
||||
|
||||
| file |
|
||||
file := (self folder / 'profile-image', 'jpg').
|
||||
file exists ifTrue: [ file asFileReference size = 0
|
||||
ifTrue:[ ^ self downloadProfileImage ].
|
||||
^ file. ].
|
||||
file := (self folder / self userName, 'jpg').
|
||||
file exists ifTrue: [ ^ file ].
|
||||
^ self downloadProfileImage
|
||||
@ -1,8 +1,4 @@
|
||||
accessing
|
||||
profileImageUrl
|
||||
|
||||
| documentTree |
|
||||
self remoteIsFound
|
||||
ifFalse:[ ^ nil ].
|
||||
documentTree := self documentTree.
|
||||
^ profileImageUrl := self class nitterProvider, (((documentTree xpath: '//div[@class="profile-card-info"]//a[@class="profile-card-avatar"]') @ 'href') stringValue copyReplaceAll: '%2F' with: '/') copyWithoutFirst
|
||||
^ profileImageUrl ifNil: [
|
||||
profileImageUrl := ((self rssFeed xmlDocument xpath: '//image/url') stringValue copyReplaceAll: '%2F' with: '/') ]
|
||||
@ -3,7 +3,7 @@ quotes
|
||||
|
||||
self messages ifEmpty: [ self getMessages ].
|
||||
^ TweetsCollection new
|
||||
messages: (self messages
|
||||
messages: (self messages messages
|
||||
select: [ :each |
|
||||
(each metadata at: 'quote') isNotEmpty and: [ each user userName = self userName] ]);
|
||||
yourself
|
||||
@ -1,22 +0,0 @@
|
||||
accessing
|
||||
quotesReportData
|
||||
|
||||
| tempDict labels xAxis |
|
||||
self quotes isEmpty
|
||||
ifTrue: [ ^ { OrderedDictionary new.
|
||||
('[''', '0', ''']').
|
||||
('[''', 'No quotes', ''']')} ].
|
||||
tempDict := self quotes asMessagesUserNamesSortedByOccurrences.
|
||||
|
||||
tempDict size > 10 ifTrue: [
|
||||
tempDict := (tempDict associations copyFrom: 1 to: 10) asOrderedDictionary ].
|
||||
|
||||
labels := tempDict keys.
|
||||
labels := labels collect: [ :profile | ($' asString), '@', profile, ($' asString) ].
|
||||
xAxis := tempDict values.
|
||||
xAxis := xAxis collect: [ :value | ($' asString), (value asString), ($' asString) ].
|
||||
^ {
|
||||
tempDict.
|
||||
('[', (',' join: xAxis), ']').
|
||||
('[', (',' join: labels), ']').
|
||||
}
|
||||
@ -1,4 +0,0 @@
|
||||
accessing
|
||||
refreshProfileImageUrl
|
||||
self profileImageUrl: nil.
|
||||
self profileImageUrl
|
||||
@ -1,5 +0,0 @@
|
||||
accessing
|
||||
remoteIsFound
|
||||
|
||||
^ (ZnClient new url: (self userNameLink asUrl); get; response) isNotFound not
|
||||
|
||||
@ -3,5 +3,5 @@ replies
|
||||
|
||||
self messages ifEmpty: [ self getMessages ].
|
||||
^ TweetsCollection new
|
||||
messages: (self tweets select: [ :each | (each metadata at: 'replie to') isNotEmpty]);
|
||||
messages: (self messages messages select: [ :each | (each metadata at: 'replie to') isNotEmpty ]);
|
||||
yourself
|
||||
@ -1,19 +0,0 @@
|
||||
accessing
|
||||
repliesByTimeSpan: divisions
|
||||
|
||||
| tweetsByTimeSpan xAxis labels |
|
||||
tweetsByTimeSpan := self collectMessages: [ self replies ] byTimeSpanSplits: divisions.
|
||||
xAxis := OrderedCollection new.
|
||||
(tweetsByTimeSpan values collect: [ :collection | collection size ]) do: [ :number |
|
||||
xAxis add: ($' asString), (number asString), ($' asString)
|
||||
].
|
||||
labels := OrderedCollection new.
|
||||
tweetsByTimeSpan keys do: [ :string |
|
||||
labels add: ($' asString), string, ($' asString)
|
||||
].
|
||||
^ {
|
||||
tweetsByTimeSpan.
|
||||
('[', (',' join: xAxis), ']').
|
||||
('[', (',' join: labels), ']').
|
||||
}
|
||||
|
||||
@ -1,23 +0,0 @@
|
||||
accessing
|
||||
repliesByWeeksTimeSpan
|
||||
|
||||
| tweetsByTimeSpan xAxis labels |
|
||||
self replies isEmpty
|
||||
ifTrue: [ ^ { OrderedDictionary new.
|
||||
('[''', '0', ''']').
|
||||
('[''', 'No replies', ''']')} ].
|
||||
tweetsByTimeSpan := self collectMessages: [ self replies ] byTimeSpanSplits: self tweetsDivisionsByWeeks.
|
||||
xAxis := OrderedCollection new.
|
||||
(tweetsByTimeSpan values collect: [ :collection | collection size ]) do: [ :number |
|
||||
xAxis add: ($' asString), (number asString), ($' asString)
|
||||
].
|
||||
labels := OrderedCollection new.
|
||||
tweetsByTimeSpan keys do: [ :string |
|
||||
labels add: ($' asString), string, ($' asString)
|
||||
].
|
||||
^ {
|
||||
tweetsByTimeSpan.
|
||||
('[', (',' join: xAxis), ']').
|
||||
('[', (',' join: labels), ']').
|
||||
}
|
||||
|
||||
@ -1,4 +0,0 @@
|
||||
private
|
||||
repliesFromImage
|
||||
"Used for testing external classes, disposable"
|
||||
^ self tweetsFromImage select: [ :tweet | (tweet metadata at: 'replie to') isNotEmpty]
|
||||
@ -1,5 +0,0 @@
|
||||
accessing
|
||||
reportingPeriod
|
||||
^ self config
|
||||
at: 'reportingPeriod'
|
||||
ifAbsentPut: [ { self messages oldest created . self messages newest created } ]
|
||||
@ -3,5 +3,5 @@ retweets
|
||||
|
||||
self messages ifEmpty: [ self getMessages ].
|
||||
^ TweetsCollection new
|
||||
messages: (self messages select: [ :each | each authorId ~= self id]);
|
||||
messages: (self messages messages select: [ :each | each authorId ~= self id]);
|
||||
yourself
|
||||
@ -1,18 +0,0 @@
|
||||
accessing
|
||||
retweetsReportData
|
||||
|
||||
| tempDict labels xAxis |
|
||||
tempDict := self retweets asMessagesUserNamesSortedByOccurrences.
|
||||
|
||||
tempDict size > 10 ifTrue: [
|
||||
tempDict := (tempDict associations copyFrom: 1 to: 10) asOrderedDictionary ].
|
||||
|
||||
labels := tempDict keys.
|
||||
labels := labels collect: [ :profile | ($' asString), '@', profile, ($' asString) ].
|
||||
xAxis := tempDict values.
|
||||
xAxis := xAxis collect: [ :value | ($' asString), (value asString), ($' asString) ].
|
||||
^ {
|
||||
tempDict.
|
||||
('[', (',' join: xAxis), ']').
|
||||
('[', (',' join: labels), ']').
|
||||
}
|
||||
@ -0,0 +1,4 @@
|
||||
accessing
|
||||
rssFeed
|
||||
|
||||
^ RSSTools createRSSFeedFor: self userNameLink , '/rss'
|
||||
@ -1,14 +1,6 @@
|
||||
accessing
|
||||
tweets
|
||||
| subcollection |
|
||||
self messages ifEmpty: [ self getMessages ].
|
||||
"TO DO: It seems that Twitter changes the user id, which we thought was unchangeable. This deals with the detected changes so far."
|
||||
|
||||
subcollection := self messages
|
||||
select: [ :each | self authorIds includes: each authorId ].
|
||||
|
||||
(#('FranciaMarquezM' 'sandralajas' 'IBetancourtCol' 'sergio_fajardo' 'ingrodolfohdez' 'CastilloMarelen') includes: self userName)
|
||||
ifFalse: [ subcollection := self messages select: [ :each | each authorId = self id ]].
|
||||
^ TweetsCollection new
|
||||
messages: subcollection;
|
||||
messages: (self messages messages select: [ :each | each authorId = self id ]);
|
||||
yourself
|
||||
@ -1,19 +0,0 @@
|
||||
accessing
|
||||
tweetsByTimeSpan: divisions
|
||||
|
||||
| tweetsByTimeSpan xAxis labels |
|
||||
tweetsByTimeSpan := self collectMessages: [ self tweets] byTimeSpanSplits: divisions.
|
||||
xAxis := OrderedCollection new.
|
||||
(tweetsByTimeSpan values collect: [ :collection | collection size ]) do: [ :number |
|
||||
xAxis add: ($' asString), (number asString), ($' asString)
|
||||
].
|
||||
labels := OrderedCollection new.
|
||||
tweetsByTimeSpan keys do: [ :string |
|
||||
labels add: ($' asString), string, ($' asString)
|
||||
].
|
||||
^ {
|
||||
tweetsByTimeSpan.
|
||||
('[', (',' join: xAxis), ']').
|
||||
('[', (',' join: labels), ']').
|
||||
}
|
||||
|
||||
@ -1,19 +0,0 @@
|
||||
accessing
|
||||
tweetsByWeeksTimeSpan
|
||||
|
||||
| tweetsByTimeSpan xAxis labels |
|
||||
tweetsByTimeSpan := self collectMessages: [ self tweets] byTimeSpanSplits: self tweetsDivisionsByWeeks.
|
||||
xAxis := OrderedCollection new.
|
||||
(tweetsByTimeSpan values collect: [ :collection | collection size ]) do: [ :number |
|
||||
xAxis add: ($' asString), (number asString), ($' asString)
|
||||
].
|
||||
labels := OrderedCollection new.
|
||||
tweetsByTimeSpan keys do: [ :string |
|
||||
labels add: ($' asString), string, ($' asString)
|
||||
].
|
||||
^ {
|
||||
tweetsByTimeSpan.
|
||||
('[', (',' join: xAxis), ']').
|
||||
('[', (',' join: labels), ']').
|
||||
}
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
accessing
|
||||
tweetsDivisionsByWeeks
|
||||
|
||||
| weeks floor divisions |
|
||||
weeks := ((self newestTweet created - self oldestTweet created) days / 7).
|
||||
floor := weeks floor.
|
||||
(weeks - floor) > 0.4
|
||||
ifTrue: [ floor = 0
|
||||
ifTrue: [ divisions := 1 ]
|
||||
ifFalse: [ divisions := floor]]
|
||||
ifFalse: [ divisions := floor + 1 ].
|
||||
^ divisions
|
||||
@ -1,4 +0,0 @@
|
||||
private
|
||||
tweetsFromImage
|
||||
"Used for testing external classes, disposable"
|
||||
^ self messages select: [ :tweet | tweet authorId = self id ]
|
||||
@ -1,12 +0,0 @@
|
||||
accessing
|
||||
updatedTweets
|
||||
| currentTweetsPage response nextPageCursor |
|
||||
|
||||
response := TweetsCollection new.
|
||||
currentTweetsPage := self collectRawTweetsFrom: self userNameLinkWithReplies upToPage: 2.
|
||||
[ (currentTweetsPage collect: [ :tweet | tweet id ]) includes: self newestTweet id ] whileFalse: [
|
||||
response addAll: currentTweetsPage.
|
||||
nextPageCursor := currentTweetsPage nextCursor.
|
||||
currentTweetsPage := self collectRawTweetsFrom: nextPageCursor upToPage: 2.
|
||||
].
|
||||
^ response
|
||||
@ -1,11 +1,7 @@
|
||||
accessing
|
||||
url
|
||||
self remoteIsFound
|
||||
ifFalse: [ ^ url := nil ].
|
||||
^ url ifNil: [ | temp |
|
||||
temp := ((self documentTree xpath: '//div[@class="profile-website"]') // 'a' @@ 'href') first.
|
||||
temp ifNil: [ ^ url := nil ].
|
||||
temp := temp replaceAll: $Ú with: $u.
|
||||
temp := temp replaceAll: $í with: $i.
|
||||
url := temp.
|
||||
url := temp asUrl.
|
||||
]
|
||||
@ -1,3 +0,0 @@
|
||||
operation
|
||||
userNameLinkWithReplies
|
||||
^ self userNameLink, '/with_replies'
|
||||
@ -1,4 +0,0 @@
|
||||
accessing
|
||||
wordcloudData
|
||||
|
||||
^ self wordcloudDataLanguage: 'en'
|
||||
@ -1,21 +0,0 @@
|
||||
accessing
|
||||
wordcloudDataLanguage: language
|
||||
|
||||
| stopwords stopwordsCapitalized occurrencesWords wordAndValue |
|
||||
stopwords := (TweetsCollection dataStore / 'commons' / ('stopwords-', language, '.txt')) contents splitOn: Character lf.
|
||||
stopwordsCapitalized := stopwords collect: [:each | each first asString asUppercase, each allButFirst asLowercase ].
|
||||
occurrencesWords := ((((self writeWordsFile contents) splitOn: ' ') asBag asDictionary)
|
||||
associations asSortedCollection: [:x :y | x value > y value]) asOrderedDictionary.
|
||||
occurrencesWords removeKeys: stopwords.
|
||||
occurrencesWords removeKeys: stopwordsCapitalized.
|
||||
occurrencesWords removeKey: ''.
|
||||
|
||||
occurrencesWords size > 50 ifTrue: [
|
||||
occurrencesWords := (occurrencesWords associations copyFrom: 1 to: 50) asOrderedDictionary ].
|
||||
|
||||
wordAndValue := OrderedCollection new.
|
||||
occurrencesWords keysAndValuesDo: [ :k :v |
|
||||
wordAndValue add: ('{name:', ($' asString), k, ($' asString), ',value:', v asString, '}')
|
||||
].
|
||||
^ {'[', ((',' join: wordAndValue) copyWithout: Character lf), ']'.
|
||||
occurrencesWords}
|
||||
@ -3,11 +3,15 @@ writeWordsFile
|
||||
"TO DO: Debug Tweet words to work better with Urls"
|
||||
| rawTweets rawWords collectionsWords count |
|
||||
self messages ifNil: [ ^ self ].
|
||||
rawTweets := self tweets.
|
||||
rawTweets := self tweets messages.
|
||||
|
||||
rawWords := OrderedCollection new.
|
||||
collectionsWords := rawTweets collect: [ :tweet | (tweet words reject: [ :w |
|
||||
w includesSubstring: (self class nitterProvider asUrl host copyWithoutAll: '.')])].
|
||||
collectionsWords := ((rawTweets select:
|
||||
[ :tweet | tweet authorId = self id ])
|
||||
collect: [ :tweet | (tweet words
|
||||
reject: [ :w |
|
||||
w includesSubstring:
|
||||
((self class nitterProvider asUrl host copyWithoutAll: '.'))]) ]).
|
||||
collectionsWords do: [ :word | rawWords addAll: word ].
|
||||
rawWords := ' ' join:rawWords.
|
||||
|
||||
|
||||
@ -1,13 +0,0 @@
|
||||
*Socialmetrica
|
||||
exportAsFileNamed: aName into: aFolder
|
||||
| exporters |
|
||||
exporters := {'png' -> #pngExporter .
|
||||
'pdf' -> #pdfExporter } asDictionary.
|
||||
exporters keysAndValuesDo: [:k :v |
|
||||
(self perform: v)
|
||||
zoomToShapes;
|
||||
noFixedShapes;
|
||||
fileName: (aFolder / (aName, '.', k)) fullName;
|
||||
export.
|
||||
].
|
||||
^ aFolder / aName, 'png'
|
||||
@ -1,3 +0,0 @@
|
||||
{
|
||||
"name" : "RSCanvas"
|
||||
}
|
||||
@ -1,5 +0,0 @@
|
||||
*Socialmetrica
|
||||
asDates
|
||||
|
||||
^ {self start.
|
||||
(self start + self duration)}
|
||||
@ -1,9 +0,0 @@
|
||||
*Socialmetrica
|
||||
asDatesString
|
||||
|
||||
| dates firstDate |
|
||||
dates := self asDatesStrings.
|
||||
firstDate := dates first splitOn: ' '.
|
||||
firstDate removeLast.
|
||||
firstDate:= '' join: firstDate.
|
||||
^ firstDate, '→', (dates second copyWithoutAll: ' ')
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in New Issue
Block a user