Updae Dataviz to merging with Roassal2-RTSunburst
Co-authored-by: yanekgil <yanekgil@gmail.com>
This commit is contained in:
akevalion
parent
378a9d1094
commit
f2ccfd15cc
|
|
@ -0,0 +1,18 @@
|
|||
"
|
||||
I store metadata for this package. These meta data are used by other tools such as the SmalllintManifestChecker and the critics Browser
|
||||
"
|
||||
Class {
|
||||
#name : #ManifestDataviz,
|
||||
#superclass : #PackageManifest,
|
||||
#category : #Dataviz
|
||||
}
|
||||
|
||||
{ #category : #'code-critics' }
|
||||
ManifestDataviz class >> ruleRBSentNotImplementedRuleV1FalsePositive [
|
||||
^ #(#(#(#RGClassDefinition #(#PublishedMedInfo)) #'2015-12-06T12:21:46.001583-05:00') )
|
||||
]
|
||||
|
||||
{ #category : #'code-critics' }
|
||||
ManifestDataviz class >> ruleRBUnclassifiedMethodsRuleV1FalsePositive [
|
||||
^ #(#(#(#RGClassDefinition #(#MedAgency)) #'2015-12-17T23:52:38.455558-05:00') )
|
||||
]
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
"
|
||||
I'm used to help in the citizen oversight of public spending.
|
||||
I load scrapped information of public Government sites (starting with
|
||||
Colombian ones) and help in its understanding by providing visualizations and other techniques.
|
||||
"
|
||||
Class {
|
||||
#name : #OpenSpending,
|
||||
#superclass : #Object,
|
||||
#instVars : [
|
||||
'contractsData'
|
||||
],
|
||||
#category : #'Dataviz-OpenSpending'
|
||||
}
|
||||
|
||||
{ #category : #accessing }
|
||||
OpenSpending >> contractsData [
|
||||
^ contractsData
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
OpenSpending >> contractsData: anObject [
|
||||
contractsData := anObject
|
||||
]
|
||||
|
||||
{ #category : #'data import' }
|
||||
OpenSpending >> listSectors [
|
||||
"I import data from aFile storaged in Comma Separated Values (CSV) format.
|
||||
Is supposed that they're taken from a query that contains ths contracting sectors
|
||||
according to https://en.wikipedia.org/wiki/UNSPSC"
|
||||
| queryResults dbSectorsClassifier |
|
||||
dbSectorsClassifier := FileLocator documents / 'DataWeek' / 'clasificador.csv'.
|
||||
queryResults := (RTTabTable new input: dbSectorsClassifier contents usingDelimiter: $,) values.
|
||||
"table removeFirstRow.
|
||||
self contractsData: table."
|
||||
^ queryResults
|
||||
]
|
||||
|
||||
{ #category : #'data import' }
|
||||
OpenSpending >> loadDataFromCSV: aFile usingDelimiter: aCharacter [
|
||||
"I import data from aFile storaged in Comma Separated Values (CSV) format.
|
||||
Is supposed that they're taken from a query that sums the sectors of the economy"
|
||||
| table |
|
||||
table := RTTabTable new input: aFile contents usingDelimiter: aCharacter.
|
||||
table removeFirstRow.
|
||||
self contractsData: table.
|
||||
]
|
||||
|
||||
{ #category : #'as yet unclassified' }
|
||||
OpenSpending >> showTreeMap [
|
||||
"I show a treemap visualization of my contract data"
|
||||
|
||||
| builder popup sectors subtotals |
|
||||
popup := RTPopup new.
|
||||
sectors := self contractsData valuesOfColumn: 1.
|
||||
subtotals := self contractsData valuesOfColumn: 2.
|
||||
popup text: [:index | (sectors at: index), String cr, (subtotals at: index) ].
|
||||
builder := RTTreeMapBuilder new.
|
||||
builder shape color: [:n | Color random].
|
||||
builder from: (1 to: (self contractsData numberOfRows)) using: [#()].
|
||||
builder weight: [:n | (self contractsData valuesOfColumn: 2) at: n ].
|
||||
builder build.
|
||||
builder view elements @ popup.
|
||||
^ builder view.
|
||||
]
|
||||
|
|
@ -0,0 +1,635 @@
|
|||
"
|
||||
Please comment me using the following template inspired by Class Responsibility Collaborator (CRC) design:
|
||||
|
||||
For the Class part: State a one line summary. For example, ""I represent a paragraph of text"".
|
||||
I represent the published medicine information from several sources for
|
||||
a particular medicine.
|
||||
|
||||
My main responsabilities are:
|
||||
- Visualize published medicine information to help in comparing several
|
||||
agencies.
|
||||
- I know how much information is published for a particular medicine.
|
||||
|
||||
My main collaborator is:
|
||||
- MedDataItem: I store several of them.
|
||||
|
||||
"
|
||||
Class {
|
||||
#name : #PublishedMedInfo,
|
||||
#superclass : #Object,
|
||||
#instVars : [
|
||||
'medName',
|
||||
'medDataItems',
|
||||
'medDataKeys',
|
||||
'medDataMatrix',
|
||||
'adminDataSize',
|
||||
'arcWidth',
|
||||
'columnLabels'
|
||||
],
|
||||
#category : #'Dataviz-MedicineInfo'
|
||||
}
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> addLabelsFrom: anArray surrounding: aView withRadio: aDistance [
|
||||
"I put a list of labels on aView which around a circle with a given radio"
|
||||
| angle |
|
||||
|
||||
1 to: anArray size do: [ :index | |label|
|
||||
angle := (index * 360/ anArray size) degreesToRadians.
|
||||
label := RTLabel new
|
||||
text: (anArray at: index) asString;
|
||||
height: aDistance / 30.
|
||||
aView
|
||||
add: (label element translateBy: (Point r: (aDistance)* 1.06 theta: angle - 0.075))
|
||||
].
|
||||
^ aView.
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> addLegendTo: aView titled: aString withData: anArray withColors: aColorPalette [
|
||||
"Adds a legend with a given title to a given view"
|
||||
|
||||
| legend |
|
||||
legend := RTLegendBuilder new.
|
||||
legend addText: aString.
|
||||
|
||||
aView
|
||||
add: legend.
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> addLineSeparatorsTo: aView withData: data columnsDistance: aDistance centerSized: internalRadio ringSized: ringSize [
|
||||
| e1 e2 ang |
|
||||
1 to: data children size do: [ :i |
|
||||
Transcript show: i.
|
||||
e1 := (RTBox new size: 1) element.
|
||||
ang := (i * 360 / data children size) degreesToRadians.
|
||||
e1 translateTo: (internalRadio * ang cos)@(internalRadio * ang sin).
|
||||
e2 := (RTBox new size: 1) element.
|
||||
e2 translateTo: ((internalRadio + ringSize) * ang cos )@ ((internalRadio + ringSize)* ang sin).
|
||||
aView
|
||||
add: e1;
|
||||
add: e2;
|
||||
add: (RTLine new width: aDistance - 4 ; color: Color veryLightGray; edgeFrom: e1 to: e2)
|
||||
]
|
||||
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> addRotatedLabelsFrom: anArray surrounding: aView withRadio: aDistance withAngularCorrection: aValue [
|
||||
"I put a list of rotated labels on aView which around a circle with a given radio"
|
||||
| angle |
|
||||
|
||||
1 to: anArray size do: [ :index | |label|
|
||||
angle := ((index * 360/ anArray size) - aValue) negated degreesToRadians.
|
||||
label := (RTRotatedLabel new
|
||||
text: (anArray at: index) asString;
|
||||
height: aDistance / 22) element.
|
||||
label translateBy: (Point r: label trachelShape notRotatedWidth/2+(aDistance)*1.06 theta: angle).
|
||||
aView add: label.
|
||||
angle := angle radiansToDegrees.
|
||||
angle := angle + ((angle between: -270 and: -90) ifTrue: [ 180 ] ifFalse: [ 0 ]).
|
||||
label trachelShape angleInDegree: angle.
|
||||
].
|
||||
^ aView.
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> adminDataSize [
|
||||
"I calculate the amount of administrative data in the matrix. By convention all administrative data starts with '*' in its name"
|
||||
|
||||
^ adminDataSize isNil
|
||||
ifTrue: [
|
||||
(self medDataKeys select: [:key | (key at: 1) = $*]) size
|
||||
]
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> adminDataSize: anObject [
|
||||
adminDataSize := anObject
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> arcWidth [
|
||||
^ arcWidth
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> arcWidth: anObject [
|
||||
arcWidth := anObject
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> colorForWithoutBiosimilar [
|
||||
"Returns a customized color for a medicament which has not biosimilar in its published info"
|
||||
|
||||
^ Color r: 218/255 g: 223/255 b: 225/255
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> colorPalette16 [
|
||||
"Returns a custom color palette for administrative data. In the data matrix, this correspond
|
||||
to the rows which name starts by '*' (first two admin data, contry and variant are already mapped in the name of each arc and don't
|
||||
start with '*'"
|
||||
|
||||
| baseColors newColorPalette index |
|
||||
baseColors := RTColorPalette qualitative
|
||||
colors: 12
|
||||
scheme:'Set3'.
|
||||
index := 1.
|
||||
newColorPalette := Array new: 16.
|
||||
baseColors do: [ :color |
|
||||
newColorPalette at: index put: color.
|
||||
index := index + 1.
|
||||
].
|
||||
"Customizing some individual colors to get a more constrasted palette, useful to distinguis in
|
||||
large information collections"
|
||||
newColorPalette
|
||||
at: 3 put: (Color r: 71/255 g:30/255 b: 253/255);
|
||||
at: 8 put: (Color r: 134/255 g:43/255 b: 80/255);
|
||||
at: 9 put: (Color r: 156/255 g:143/255 b: 50/255);
|
||||
at: 11 put: (Color r: 164/255 g:44/255 b: 107/255);
|
||||
at: 13 put: (Color r:0.75 g:0.22 b:0.17);
|
||||
at: 14 put: (Color r:0.42 g:0.48 b:0.54);
|
||||
at: 15 put: (Color r:0.83 g:0.33 b:0);
|
||||
at: 16 put: (Color r: 211/255 g:26/255 b: 239/255).
|
||||
^ newColorPalette.
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> colorPalette26 [
|
||||
"Returns a custom color palette for prescrition and use data. In the data matrix, this correspond
|
||||
to the rows which name doesn't start by '*' (first two admin data, contry and variant are already mapped in the name of each arc and doesn't
|
||||
start with '*'"
|
||||
|
||||
| baseColors newColorPalette index |
|
||||
baseColors := RTColorPalette qualitative
|
||||
colors: 20
|
||||
scheme:'FlatUI1'.
|
||||
index := 1.
|
||||
newColorPalette := Array new: 26.
|
||||
baseColors do: [ :color |
|
||||
newColorPalette at: index put: color.
|
||||
index := index + 1.
|
||||
].
|
||||
newColorPalette
|
||||
at: 2 put: (Color r:0.65 g:0.22 b:0.23);
|
||||
at: 4 put: (Color r:1 g:0.72 b:0.1);
|
||||
at: 6 put: (Color r:255/255 g:230/255 b:141/255);
|
||||
at: 8 put: (Color r:178/255 g:24/255 b:17/255);
|
||||
at: 14 put: (Color r:255/255 g:214/255 b:208/255);
|
||||
at: 18 put: (Color r:218/255 g:0/255 b:132/255);
|
||||
at: 21 put: (Color r:223/255 g:93/255 b:255/255);
|
||||
at: 22 put: (Color r:0.29 g:0.47 b:0.75);
|
||||
at: 23 put: (Color r:0.64 g:0.87 b:0.82);
|
||||
at: 24 put: (Color r:131/255 g:178/255 b:10/255);
|
||||
at: 25 put: (Color r:0.91 g:0.83 b:038);
|
||||
at: 26 put: (Color r:0.42 g:0.48 b:054).
|
||||
^ newColorPalette.
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> columnLabels [
|
||||
^ columnLabels
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> columnLabels: anObject [
|
||||
columnLabels := anObject
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> exploreDataForCountriesFromRow: firstRowIndex upTo: lastRowIndex coloredWith: aColorPalette centerTitled: aString arcWidth: aNumber [
|
||||
"I show a subset of the matrix rows using a sunburst visualization variation"
|
||||
|
||||
self arcWidth: aNumber.
|
||||
^ self
|
||||
exploreMatrix: (self medDataMatrix copyFrom: firstRowIndex to: lastRowIndex)
|
||||
by: 'country'
|
||||
coloredWith: aColorPalette
|
||||
centerSized: 200
|
||||
separationSized: 5
|
||||
slicedFrom: firstRowIndex
|
||||
centerTitled: aString
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> exploreMatrix: aMatrix by: type coloredWith: aColorPalette centerSized: centerSize separationSized: separationSize slicedFrom: anInteger centerTitled: aString [
|
||||
"I create a sunburst diagram for matrix alike information using aPalette for each of the rings on the sunburst.
|
||||
Information of the popup is taken according to the type of exploration that is being done in the matrix, which can
|
||||
done by the country which is publishing the info or by properties that are bein published.
|
||||
This code has a lot of parameters and maybe needs serious refactoring"
|
||||
|
||||
| b data externalCircle internalCircle medLabel index |
|
||||
|
||||
data := self matrixToTreeReversed: aMatrix.
|
||||
b := RTSunburstBuilder new.
|
||||
b strategy
|
||||
centerWidth: centerSize;
|
||||
hasCenter: false;
|
||||
arcWidth: self arcWidth;
|
||||
radialSpacing: 50.
|
||||
internalCircle := RTEllipse new
|
||||
width: centerSize*2; height: centerSize*2;
|
||||
color: Color white trans;
|
||||
borderColor: Color veryLightGray;
|
||||
borderWidth: 1.
|
||||
externalCircle := RTEllipse new
|
||||
width: internalCircle element width + (2 * aMatrix size * (self arcWidth + 1));
|
||||
height: internalCircle element width + (2 * aMatrix size * (self arcWidth + 1));
|
||||
color: Color white trans;
|
||||
borderColor: Color veryLightGray;
|
||||
borderWidth: 1.
|
||||
b view add: externalCircle element.
|
||||
b view add: internalCircle element.
|
||||
medLabel := RTLabel new
|
||||
text: aString, String cr, self medName;
|
||||
color: Color black;
|
||||
height: (internalCircle element width)/20 asInteger.
|
||||
b view add: (medLabel element translateBy: (internalCircle element center)).
|
||||
b radialSpacing: 1.
|
||||
index := 1.
|
||||
b shape color: [ :node |
|
||||
node header asString size > 1
|
||||
ifTrue: [ Color black ]
|
||||
ifFalse: [
|
||||
node header asInteger isZero
|
||||
ifTrue: [ Color white ]
|
||||
ifFalse: [ node header asInteger = 2
|
||||
ifTrue: [ self colorForWithoutBiosimilar ]
|
||||
ifFalse: [ aColorPalette at: node level ]
|
||||
]
|
||||
]
|
||||
].
|
||||
b interaction noInteractions.
|
||||
type asLowercase = 'country' ifTrue: [
|
||||
b interaction
|
||||
addInteraction:
|
||||
(RTPopup text: [:d | (self medDataKeys at: ((aMatrix size) - d level + anInteger + 1)), '-> ', d header]).
|
||||
].
|
||||
type asLowercase = 'property' ifTrue: [
|
||||
b interactions
|
||||
add:
|
||||
(RTPopup text: [:d | self labelsForCountries at: (d level - 1)]).
|
||||
].
|
||||
b explore: data using: #children.
|
||||
b strategy hasCenter: false.
|
||||
b build.
|
||||
b view @ RTZoomableBoxView.
|
||||
self
|
||||
addLineSeparatorsTo: b view
|
||||
withData: data
|
||||
columnsDistance: separationSize
|
||||
centerSized: (internalCircle element width / 2)
|
||||
ringSized: ((externalCircle element width/2) - (internalCircle element width/2)).
|
||||
^ {'theView' -> b view . 'externalRadio' -> (externalCircle element width/2)} asDictionary.
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> labelsForAdminProperties [
|
||||
"I got the information about the administrative properties that are being published by
|
||||
all countries public health agencies.
|
||||
This information is a subset of medDataKeys"
|
||||
|
||||
^ self medDataKeys copyFrom: 3 to: self adminDataSize + 2.
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> labelsForCountries [
|
||||
"I got the information about the country who is publishing the medicine info.
|
||||
This information is stored in the first row of the medDataMatrix.
|
||||
Countries use the ISO 3166-1 alpha 3 standard."
|
||||
|
||||
| labels allCountries |
|
||||
allCountries := (self medDataMatrix at: 1).
|
||||
labels := OrderedCollection new.
|
||||
1 to: allCountries size by: 2 do: [:index |
|
||||
labels add: (allCountries at: index).
|
||||
].
|
||||
^ labels reversed.
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> labelsForCountriesAndVariants [
|
||||
"I got the information about the country and variant of medicine info that is being published by a
|
||||
country public health agency.
|
||||
This information is stored in the first two rows of the medDataMatrix.
|
||||
Countries use the ISO 3166-1 alpha 3 standard and for the medicine variant we use complete words
|
||||
in the CVS but select only the first letter for visualization purposes, so the first letter is
|
||||
supposed to be different on each variant."
|
||||
|
||||
| labels countries variants |
|
||||
countries := self medDataMatrix at: 1.
|
||||
variants := self medDataMatrix at: 2.
|
||||
labels := Array new: countries size.
|
||||
1 to: countries size do: [:index |
|
||||
labels
|
||||
at: index
|
||||
put: (countries at: (countries size + 1 - index)),
|
||||
'-',
|
||||
(((variants at: (variants size + 1 - index)) copyFrom: 1 to: 1) asLowercase)
|
||||
].
|
||||
^ labels
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> labelsForPUProperties [
|
||||
"I got the information about the properties that are being published by all countries public health agencies.
|
||||
This information is a subset of medDataKeys"
|
||||
|
||||
^ self medDataKeys copyFrom: 3 + self adminDataSize to: self medDataKeys size.
|
||||
]
|
||||
|
||||
{ #category : #'data import' }
|
||||
PublishedMedInfo >> loadDataFromCSV: aFile usingDelimiter: aCharacter [
|
||||
"I import data from aFile storaged in Comma Separated Values (CSV) format.
|
||||
Is supposed that:
|
||||
- First row contains medicine name information.
|
||||
- First column contains the attributes that are being evaluated
|
||||
- Other columns contain the result of such evaluation as numerical values"
|
||||
| table |
|
||||
table := RTTabTable new input: aFile contents usingDelimiter: aCharacter.
|
||||
self medName: ((table valuesOfColumn: 2) at: 1).
|
||||
table removeFirstRow.
|
||||
self medDataKeys: table firstColumn.
|
||||
table removeColumn: 1.
|
||||
self medDataMatrix: table values.
|
||||
|
||||
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> matrixDataForAdmin [
|
||||
"I extract all the administrative information from the medDataMatrix as a transposed matrix, so it can be used
|
||||
as an input for the matrixSunburst visualizations.
|
||||
By convention, this data is stored after the admin data for each drug has ended"
|
||||
|
||||
| submatrixForAdmin numberOfColums transposedMatrix transposedRow |
|
||||
submatrixForAdmin := self medDataMatrix copyFrom: 3 to: (self adminDataSize + 2).
|
||||
numberOfColums := (submatrixForAdmin at: 1) size.
|
||||
transposedMatrix := OrderedCollection new.
|
||||
1 to: (numberOfColums) by: 2 do: [ :columIndex |
|
||||
transposedRow := Array new: submatrixForAdmin size.
|
||||
1 to: submatrixForAdmin size do: [:index |
|
||||
transposedRow at: index put: ((submatrixForAdmin at: index) at: columIndex)
|
||||
].
|
||||
transposedMatrix add: transposedRow
|
||||
].
|
||||
^ transposedMatrix asArray
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> matrixDataForPU [
|
||||
"I extract all the prescription and use data from the medDataMatrix as a transposed matrix, so it can be used
|
||||
as an input for the matrixSunburst visualizations.
|
||||
By convention, this data is stored after the admin data for each drug has ended"
|
||||
|
||||
| submatrixForPU numberOfColums transposedMatrix transposedRow |
|
||||
submatrixForPU := self medDataMatrix copyFrom: 3 + self adminDataSize to: self medDataMatrix size.
|
||||
numberOfColums := (submatrixForPU at: 1) size.
|
||||
transposedMatrix := OrderedCollection new.
|
||||
1 to: (numberOfColums) by: 2 do: [ :columIndex |
|
||||
transposedRow := Array new: submatrixForPU size.
|
||||
1 to: submatrixForPU size do: [:index |
|
||||
transposedRow at: index put: ((submatrixForPU at: index) at: columIndex)
|
||||
].
|
||||
transposedMatrix add: transposedRow
|
||||
].
|
||||
^ transposedMatrix asArray
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> matrixSunburstForAdminDataByCountry [
|
||||
"Shows a matrix sunburst for the drug administrative data in several public drug agencies, using the
|
||||
properties of the medicine as tows and the countries that release the info (or not) as columns.
|
||||
By convention, this data starts at the 3rd row and ends in 2 + self adminDataSize."
|
||||
|
||||
| temp view radio |
|
||||
temp := self
|
||||
exploreDataForCountriesFromRow: 3
|
||||
upTo: 1 + self adminDataSize
|
||||
coloredWith: self colorPalette16
|
||||
centerTitled: ''
|
||||
arcWidth: 15.
|
||||
view := temp at: 'theView'.
|
||||
radio := temp at: 'externalRadio'.
|
||||
^ self
|
||||
addLabelsFrom: self labelsForCountriesAndVariants
|
||||
surrounding: view
|
||||
withRadio: radio*1.05.
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> matrixSunburstForAdminDataByProperty [
|
||||
"Shows a matrix sunburst for the drug administrative data in several public drug agencies, using the
|
||||
properties of the medicine as columns and the countries as rows."
|
||||
|
||||
| temp view radio |
|
||||
temp := self
|
||||
exploreMatrix: self matrixDataForAdmin
|
||||
by: 'property'
|
||||
coloredWith: self colorPalette16
|
||||
centerSized: 200
|
||||
separationSized: 6
|
||||
slicedFrom: nil
|
||||
centerTitled: ''.
|
||||
view := temp at: 'theView'.
|
||||
radio := temp at: 'externalRadio'.
|
||||
^ self
|
||||
addRotatedLabelsFrom: self labelsForAdminProperties
|
||||
surrounding: view
|
||||
withRadio: radio * 1.05
|
||||
withAngularCorrection: 14.5
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> matrixSunburstForAdminDataWithColorConvention [
|
||||
"Shows a matrix sunburst for the drug administrative data in several public drug agencies"
|
||||
|
||||
| composer sunburst |
|
||||
composer := RTComposer new.
|
||||
sunburst := self matrixSunburstForAdminDataByCountry.
|
||||
composer group: #sunburst.
|
||||
composer view.
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> matrixSunburstForPUDataByCountry [
|
||||
"Shows a matrix sunburst for the drug prescripton and use data in several public drug agencies.
|
||||
For convention, this data stats at the row 3 + self adminDataSize and ends at the last one of the matrix"
|
||||
| temp view radio |
|
||||
temp:= self
|
||||
exploreDataForCountriesFromRow: 3 + self adminDataSize
|
||||
upTo: self medDataKeys size
|
||||
coloredWith: self colorPalette26
|
||||
centerTitled: ''
|
||||
arcWidth: 15.
|
||||
view := temp at: 'theView'.
|
||||
radio := temp at: 'externalRadio'.
|
||||
^ self
|
||||
addLabelsFrom: self labelsForCountriesAndVariants
|
||||
surrounding: view
|
||||
withRadio: radio*1.05.
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> matrixSunburstForPUDataByProperty [
|
||||
"Shows a matrix sunburst for the drug administrative data in several public drug agencies, using the
|
||||
properties of the medicine as columns and the countries as rows."
|
||||
|
||||
| temp view radio |
|
||||
temp := self
|
||||
exploreMatrix: self matrixDataForPU
|
||||
by: 'property'
|
||||
coloredWith: self colorPalette16
|
||||
centerSized: 200
|
||||
separationSized: 6
|
||||
slicedFrom: nil
|
||||
centerTitled: ''.
|
||||
view := temp at: 'theView'.
|
||||
radio := temp at: 'externalRadio'.
|
||||
^ self
|
||||
addRotatedLabelsFrom: self labelsForPUProperties
|
||||
surrounding: view
|
||||
withRadio: radio * 1.05
|
||||
withAngularCorrection: 9
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> matrixToTree: aMatrix [
|
||||
"Converts aMatrix to a tree so it can be visualized using a sunburst"
|
||||
|
||||
| currentNode tree column |
|
||||
tree := GrafoscopioNode new level: 1.
|
||||
(aMatrix at: 1) do: [ :label |
|
||||
tree addNode: (GrafoscopioNode new header: label)
|
||||
].
|
||||
column := 1.
|
||||
tree children do: [ :child |
|
||||
currentNode := child.
|
||||
2 to: (aMatrix size) do: [:row |
|
||||
currentNode addNode: (GrafoscopioNode new header: ((aMatrix at: row) at: column)).
|
||||
currentNode := currentNode children at: 1.
|
||||
].
|
||||
column := column + 1.
|
||||
].
|
||||
^ tree.
|
||||
]
|
||||
|
||||
{ #category : #utility }
|
||||
PublishedMedInfo >> matrixToTreeReversed: aMatrix [
|
||||
"Converts aMatrix to a tree so it can be visualized using a sunburst.
|
||||
Root of the tree is near to the last row, so the most exterior rings correspond to first rows,
|
||||
and inner most ones correspond to the last rows"
|
||||
|
||||
| currentNode tree column |
|
||||
tree := GrafoscopioNode new level: 1.
|
||||
(aMatrix at: aMatrix size) do: [ :label |
|
||||
tree addNode: (GrafoscopioNode new header: label)
|
||||
].
|
||||
column := 1.
|
||||
tree children do: [ :child |
|
||||
currentNode := child.
|
||||
(aMatrix size -1 ) to: 1 by: -1 do: [:row |
|
||||
currentNode addNode: (GrafoscopioNode new header: ((aMatrix at: row) at: column)).
|
||||
currentNode := currentNode children at: 1.
|
||||
].
|
||||
column := column + 1.
|
||||
].
|
||||
^ tree.
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> medDataItems [
|
||||
^ medDataItems
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> medDataItems: anObject [
|
||||
medDataItems := anObject
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> medDataKeys [
|
||||
^ medDataKeys
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> medDataKeys: anObject [
|
||||
medDataKeys := anObject
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> medDataMatrix [
|
||||
^ medDataMatrix
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> medDataMatrix: anObject [
|
||||
medDataMatrix := anObject
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> medName [
|
||||
^ medName
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
PublishedMedInfo >> medName: anObject [
|
||||
medName := anObject
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> showAdminDataColorConventions [
|
||||
"I create a legend showing the color palette and properties used in the matrix sunburst for administrative drug data"
|
||||
|
||||
| legendBox adminDataKeys |
|
||||
|
||||
legendBox := RTLegendBuilder new.
|
||||
legendBox addText: 'Convenciones' asUppercase.
|
||||
adminDataKeys := self medDataKeys copyFrom: 3 to: self adminDataSize + 1.
|
||||
1 to: adminDataKeys size do: [ :i |
|
||||
legendBox addColor: (self colorPalette16 at: (self colorPalette16 size + 1 - i)) text: ((adminDataKeys at: i) copyReplaceAll: '* ' with: '').
|
||||
].
|
||||
legendBox addColor: self colorForWithoutBiosimilar text: 'Sin biosimilar aprobado'.
|
||||
legendBox addColor: Color white text: 'Sin informacion publicada'.
|
||||
legendBox build.
|
||||
^ legendBox view
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> showCountryColorConventions [
|
||||
"I create a legend showing the color palette and countries used in the matrix sunburst for prescription drug data"
|
||||
|
||||
| legendBox |
|
||||
|
||||
legendBox := RTLegendBuilder new.
|
||||
legendBox addText: 'Convenciones' asUppercase.
|
||||
self labelsForCountries size to: 1 by: -1 do: [ :i |
|
||||
legendBox addColor: (self colorPalette16 at: i + 1) text: (self labelsForCountries at: i).
|
||||
].
|
||||
legendBox addColor: self colorForWithoutBiosimilar text: 'Sin biosimilar aprobado'.
|
||||
legendBox addColor: Color white text: 'Sin informacion publicada'.
|
||||
legendBox build.
|
||||
^ legendBox view
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
PublishedMedInfo >> showPUDataColorConventions [
|
||||
"I create a legend showing the color palette and properties used in the matrix sunburst for prescription drug data.
|
||||
By convention this data starst where admin data ends and two first rows are 'special' admin data: name and variant of
|
||||
medicament which are already mapped on"
|
||||
|
||||
| legendBox |
|
||||
|
||||
legendBox := RTLegendBuilder new.
|
||||
legendBox addText: 'Convenciones' asUppercase.
|
||||
|
||||
1 to: (self medDataKeys size) - (self adminDataSize + 2) do: [ :i |
|
||||
legendBox addColor: (self colorPalette26 at: (self colorPalette26 size + 1 - i)) text: ((self medDataKeys at: self adminDataSize + 2 + i) copyReplaceAll: '* ' with: '').
|
||||
].
|
||||
legendBox addColor: self colorForWithoutBiosimilar text: 'Sin biosimilar aprobado'.
|
||||
legendBox addColor: Color white text: 'Sin informacion publicada'.
|
||||
legendBox build.
|
||||
^ legendBox view
|
||||
]
|
||||
|
|
@ -11,7 +11,11 @@ Class {
|
|||
'date',
|
||||
'message',
|
||||
'profile',
|
||||
'url'
|
||||
'url',
|
||||
'mentions',
|
||||
'links',
|
||||
'hashtags',
|
||||
'type'
|
||||
],
|
||||
#category : #'Dataviz-Twitter'
|
||||
}
|
||||
|
|
@ -26,6 +30,46 @@ Tweet >> date: anObject [
|
|||
date := anObject
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
Tweet >> hashtags [
|
||||
^ hashtags
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
Tweet >> hashtags: anObject [
|
||||
hashtags := anObject
|
||||
]
|
||||
|
||||
{ #category : #'data scrapping' }
|
||||
Tweet >> impact [
|
||||
"Computes the maximum amount of followers that a tweet can reach, including the retweets.
|
||||
It doesn't take into account that retweeters can share followers and in future versions
|
||||
this should be computed, even if we don't have a full list of followers.
|
||||
Also the computing time could be sustancially improved by extracting only followers data
|
||||
from the profiles instead of scrapping all and by having a better cache which includes
|
||||
prescrapped data.
|
||||
'Real time' is not the main issue here, but 'slow data'.
|
||||
For example we could include snapshots of the retweeters profiles and to aks for a
|
||||
commented retweet to reach new followers"
|
||||
|
||||
| retweetsHtml retweetsTree retweeters impactData |
|
||||
|
||||
retweetsHtml := ZnClient new get: 'http://mutabit.com/deltas/repos.fossil/datapolis/doc/tip/Data/Sources/Single/', ((self url splitOn: '//') at: 2), '/retweets.html'.
|
||||
|
||||
retweetsTree := Soup fromString: retweetsHtml.
|
||||
|
||||
retweeters := (retweetsTree findAllTagsByClass: 'username')
|
||||
collect: [:each | each text copyReplaceAll: '@' with: ''].
|
||||
|
||||
impactData := Dictionary new.
|
||||
|
||||
impactData
|
||||
at: 'retweeters' put: retweeters size;
|
||||
at: 'reach' put: (retweeters collect: [:each | (TwitterProfile new scrapDataForProfile: each) followers]) sum.
|
||||
|
||||
^ impactData.
|
||||
]
|
||||
|
||||
{ #category : #'data scrapping' }
|
||||
Tweet >> impactFor: aTweetUrl [
|
||||
"Computes the maximum amount of followers that a tweet can reach, including the retweets.
|
||||
|
|
@ -51,7 +95,7 @@ Tweet >> impactFor: aTweetUrl [
|
|||
|
||||
impactData
|
||||
at: 'retweeters' put: retweeters size;
|
||||
at: 'reach' put: (retweeters collect: [:each | (TwitterProfile new scrapDataFromProfile: each) followers]) sum.
|
||||
at: 'reach' put: (retweeters collect: [:each | (TwitterProfile new scrapFollowersForProfile: each)]) sum.
|
||||
|
||||
^ impactData.
|
||||
]
|
||||
|
|
@ -94,6 +138,62 @@ RTNest new on: background nest: els.
|
|||
^ background
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
Tweet >> links [
|
||||
^ links
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
Tweet >> links: anObject [
|
||||
links := anObject
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
Tweet >> mentions [
|
||||
^ mentions
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
Tweet >> mentions: anObject [
|
||||
mentions := anObject
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
Tweet >> mentionsClusterSeparated: aDistance inView: aView [
|
||||
"I return a graphic of all avatars mentioned in a tweet, layered as a cluster a separated
|
||||
by aDistance"
|
||||
|
||||
| container avatars profiles |
|
||||
|
||||
"Profiles: Create profiles collecting information for all users mentioned in the tweet"
|
||||
profiles := OrderedCollection new.
|
||||
self mentions do: [:eachMention |
|
||||
profiles add: (TwitterProfile new scrapDataForProfile: eachMention)].
|
||||
|
||||
"Avatars: Use the profiles to create a collection of avatar images which are draggable
|
||||
elements on a view"
|
||||
avatars := OrderedCollection new.
|
||||
profiles do: [:eachProfile |
|
||||
avatars add: (RTBitmap new form: eachProfile avatar) element ].
|
||||
RTCircleLayout new
|
||||
initialRadius: (avatars size)*aDistance;
|
||||
on: avatars.
|
||||
|
||||
container := (RTBox new color: Color transparent) element .
|
||||
RTNest new on: container nest: avatars.
|
||||
|
||||
"Adding avatars and its labels"
|
||||
aView addAll: avatars.
|
||||
aView add: container.
|
||||
|
||||
1 to: (avatars size) do: [:i |
|
||||
(avatars at: i) @ (RTLabelled new
|
||||
text: (('@',(self mentions at: i)));
|
||||
fontSize: 35;
|
||||
color: (Color black))].
|
||||
^ container
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
Tweet >> message [
|
||||
^ message
|
||||
|
|
@ -114,33 +214,199 @@ Tweet >> profile: anObject [
|
|||
profile := anObject
|
||||
]
|
||||
|
||||
{ #category : #'data scrapping' }
|
||||
Tweet >> scrapDataFromUrl: aTweetUrl [
|
||||
"Scraps most of the data in the page of a aTweetUrl. Most of the tweets are prestored now, but in the future
|
||||
it will combine prestored data and live streaming data"
|
||||
|
||||
| tweetSourceHtml tweetSourceTree hour dateTmp day month year|
|
||||
tweetSourceHtml := ZnClient new get:
|
||||
'http://mutabit.com/deltas/repos.fossil/datapolis/doc/tip/Data/Sources/Single/',
|
||||
((aTweetUrl splitOn: '//') at: 2), '/tweet.html'.
|
||||
|
||||
tweetSourceTree := Soup fromString: tweetSourceHtml.
|
||||
self url: aTweetUrl .
|
||||
self profile: ((aTweetUrl splitOn: '/') at: 4).
|
||||
self message: ((((tweetSourceTree findAllTagsByClass: 'tweet') at: 1) children at: 6) children at: 2).
|
||||
self mentions: ((self message findAllTagsByClass: 'twitter-atreply')
|
||||
collect: [:each | (each attributeAt: 'href') copyReplaceAll: '/' with: '']).
|
||||
self hashtags: ((self message findAllTagsByClass: 'twitter-hashtag') collect: [:each | each text]).
|
||||
self links: ((self message findAllTagsByClass: 'twitter-timeline-link')
|
||||
collect: [:each | each attributeAt: 'data-url']).
|
||||
hour := ((((tweetSourceTree findAllTagsByClass: 'tweet') at: 1) children at: 8) children at: 2).
|
||||
dateTmp := (((((tweetSourceTree findAllTagsByClass: 'tweet') at: 1) children at: 8) children at: 4) text) splitOn: ' '.
|
||||
day := dateTmp at: 3.
|
||||
month := dateTmp at: 4.
|
||||
year := '20', (dateTmp at: 5).
|
||||
self date: (year, month, day, hour text, hour nextSibling text) asDateAndTime.
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
Tweet >> silenceMap [
|
||||
"Creates a visualization of how long a tweet has not been answered (any kind of answer: not favs, not RT, no nothing), as
|
||||
'circles of silence' "
|
||||
Tweet >> showInView: aView [
|
||||
"shows the general information (author, author's avatar and text) of the tweet in a view"
|
||||
|
||||
| v circle ex1 ex2 line retweets reach diameter radious origin |
|
||||
| tweetPanel tweetText holder avatar |
|
||||
|
||||
"Tweet text"
|
||||
(self message text size > 50)
|
||||
ifTrue:[
|
||||
tweetText := RTLabel new
|
||||
text: (self splitByLines at: 1), String cr,
|
||||
(self splitByLines at: 2), String cr, String cr,
|
||||
(self date asString)]
|
||||
ifFalse: [tweetText := RTLabel new
|
||||
text: (self splitByLines at: 1), String cr, String cr,
|
||||
(self date asString) ].
|
||||
tweetText height: 35.
|
||||
tweetText := tweetText element.
|
||||
|
||||
"Avatar"
|
||||
avatar := (RTBitmap new form:
|
||||
((TwitterProfile new scrapDataForProfile: self profile) avatar)) element.
|
||||
avatar translateBy: (-30*(30)@0).
|
||||
|
||||
"Putting tweet elements together"
|
||||
tweetPanel := OrderedCollection new.
|
||||
tweetPanel add: avatar; add: tweetText.
|
||||
holder := (RTBox new color: Color transparent) element @ RTDraggable.
|
||||
RTNest new on: holder nest: tweetPanel.
|
||||
|
||||
"Adding elements and labels to the view"
|
||||
aView add: tweetText.
|
||||
aView add: avatar.
|
||||
aView add: holder.
|
||||
avatar @ (RTLabelled new
|
||||
text: '@', (self profile);
|
||||
fontSize: 35;
|
||||
color: Color black;
|
||||
top;
|
||||
offsetOnEdge: 1 ).
|
||||
^ holder.
|
||||
]
|
||||
|
||||
diameter := 60.
|
||||
origin := (100@100).
|
||||
radious := diameter / 2.
|
||||
v := RTView new.
|
||||
{ #category : #'data visualization' }
|
||||
Tweet >> showInView: aView sized: aSize [
|
||||
"shows the general information (author, author's avatar and text) of the tweet in a view"
|
||||
|
||||
circle := (RTEllipse new size: diameter; borderColor: Color black; color: Color transparent) element.
|
||||
ex1 := RTBox element.
|
||||
ex2 := RTBox element.
|
||||
circle translateTo: origin.
|
||||
ex1 translateTo: ((radious negated @ 0) + circle center).
|
||||
ex2 translateTo: (radious @ 0) + circle center.
|
||||
line := (RTLine new; color: Color black) edgeFrom: ex1 to: ex2.
|
||||
v add: circle; add: line.
|
||||
| tweetPanel tweetText holder avatar |
|
||||
|
||||
"Tweet text"
|
||||
(self message text size > 50)
|
||||
ifTrue:[
|
||||
tweetText := RTLabel new
|
||||
text: (self splitByLines at: 1), String cr,
|
||||
(self splitByLines at: 2), String cr, String cr,
|
||||
(self date asString)]
|
||||
ifFalse: [tweetText := RTLabel new
|
||||
text: (self splitByLines at: 1), String cr, String cr,
|
||||
(self date asString) ].
|
||||
tweetText height: aSize.
|
||||
tweetText := tweetText element.
|
||||
|
||||
"Avatar"
|
||||
avatar := (RTBitmap new form:
|
||||
((TwitterProfile new scrapDataForProfile: self profile) avatar)) element.
|
||||
avatar translateBy: (-30*aSize@0).
|
||||
|
||||
"Putting tweet elements together"
|
||||
tweetPanel := OrderedCollection new.
|
||||
tweetPanel add: avatar; add: tweetText.
|
||||
holder := (RTBox new color: Color transparent) element @ RTDraggable.
|
||||
RTNest new on: holder nest: tweetPanel.
|
||||
|
||||
"Adding elements and labels to the view"
|
||||
aView add: tweetText.
|
||||
aView add: avatar.
|
||||
aView add: holder.
|
||||
avatar @ (RTLabelled new
|
||||
text: '@', (self profile);
|
||||
fontSize: 35;
|
||||
color: Color black;
|
||||
top;
|
||||
offsetOnEdge: 1 ).
|
||||
^ holder.
|
||||
]
|
||||
|
||||
retweets := (RTLabel text: 5) element.
|
||||
reach := (RTLabel text: '3.1k') element.
|
||||
v add: retweets; add: reach.
|
||||
retweets translateTo: (0 @ -12) + circle center.
|
||||
reach translateTo: (0 @ 12) + circle center.
|
||||
^ v.
|
||||
{ #category : #'data visualization' }
|
||||
Tweet >> silenceMapFor: arg1 [
|
||||
| tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 tmp7 tmp8 tmp9 tmp11 tmp13 tmp15 tmp17 tmp19 tmp21 |
|
||||
self scrapDataFromUrl: arg1.
|
||||
tmp5 := (Date today - self date) days.
|
||||
tmp7 := self impactFor: arg1.
|
||||
tmp11 := RTBox new.
|
||||
tmp11 color: Color red.
|
||||
tmp3 := tmp11 size: 200.
|
||||
tmp3 := tmp3 element.
|
||||
tmp3 translateBy: (tmp5 * -50) @ 0.
|
||||
tmp1 := RTView new.
|
||||
tmp6 := self mentionsClusterSeparated: 70 inView: tmp1.
|
||||
tmp2 := self showInView: tmp1 sized: 35.
|
||||
tmp2 translateBy: (tmp5 * -25) @ -600.
|
||||
tmp1 add: tmp3.
|
||||
tmp4 := RTEdge from: tmp6 to: tmp3.
|
||||
tmp13 := RTGradientColoredLine new.
|
||||
tmp13
|
||||
colors: (Array with: (Color white alpha: 0.3) with: (Color red alpha: 0.9));
|
||||
precision: 100;
|
||||
width: 20.
|
||||
tmp1 add: tmp4 + tmp13 gradientColorShape.
|
||||
tmp15 := RTLabel new.
|
||||
tmp15 text: tmp5 asString , ' días sin respuesta'.
|
||||
tmp8 := (tmp15 height: tmp5 * 2.5) element @ RTDraggable.
|
||||
tmp8 translateBy: (tmp5 * -25) @ -105.
|
||||
tmp17 := RTLabel new.
|
||||
tmp17
|
||||
text: 'al ' , Date today asString , ' y contando...';
|
||||
height: tmp5 * 1.
|
||||
tmp9 := (tmp17 color: Color gray) element @ RTDraggable.
|
||||
tmp9 translateBy: (tmp5 * -15) @ 50.
|
||||
tmp1
|
||||
add: tmp8;
|
||||
add: tmp9.
|
||||
tmp19 := RTLabelled new.
|
||||
tmp19
|
||||
text: (tmp7 at: 'retweeters') asString , ' retweets';
|
||||
fontSize: tmp5 * 2.
|
||||
tmp3 @ (tmp19 color: Color gray).
|
||||
tmp21 := RTLabelled new.
|
||||
tmp21
|
||||
text: (tmp7 at: 'reach') asString , ' lectores' , String cr , '(max)';
|
||||
fontSize: tmp5 * 2;
|
||||
color: Color gray.
|
||||
tmp3 @ tmp21 below.
|
||||
tmp1 view canvas focusOnCenterScaled.
|
||||
^ tmp1 @ RTDraggableView
|
||||
]
|
||||
|
||||
{ #category : #'as yet unclassified' }
|
||||
Tweet >> splitByLines [
|
||||
"Splits the message text of the tweet in lines, in case of being too long for a better rendering
|
||||
in silenceMap and other views."
|
||||
|
||||
| summa i charLimit words wordSizes lines |
|
||||
|
||||
words := self message text splitOn: ' '.
|
||||
wordSizes := words collect: [:eachWord | eachWord size].
|
||||
i := 1.
|
||||
summa := 0.
|
||||
charLimit := 50.
|
||||
[summa < charLimit] whileTrue: [
|
||||
summa := (wordSizes at: i) + summa.
|
||||
i := i +1].
|
||||
lines := OrderedCollection new.
|
||||
lines add: (Character space join: (words copyFrom: 1 to: i)).
|
||||
lines add: (Character space join: (words copyFrom: i + 1 to: words size)).
|
||||
^ lines
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
Tweet >> type [
|
||||
^ type
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
Tweet >> type: anObject [
|
||||
type := anObject
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
|
|
|
|||
|
|
@ -0,0 +1,345 @@
|
|||
"
|
||||
Please comment me using the following template inspired by Class Responsibility Collaborator (CRC) design:
|
||||
|
||||
I'm Tweets a helper class to work with collections of tweet objects.
|
||||
|
||||
For the Responsibility part: Three sentences about my main responsibility, what I'm doing, what services do I offer.
|
||||
|
||||
For the Collaborators Part: State my main collaborators and one line about how I interact with them.
|
||||
|
||||
Public API and Key Messages
|
||||
|
||||
- message one
|
||||
- message two
|
||||
- what is the way to create instances is a plus.
|
||||
|
||||
One simple example is simply gorgeous.
|
||||
|
||||
Internal Representation and Key Implementation Points.
|
||||
|
||||
Instance Variables
|
||||
tweets: <Object>
|
||||
|
||||
|
||||
Implementation Points
|
||||
"
|
||||
Class {
|
||||
#name : #TweetsCollection,
|
||||
#superclass : #Object,
|
||||
#instVars : [
|
||||
'tweets'
|
||||
],
|
||||
#category : #'Dataviz-Twitter'
|
||||
}
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
TweetsCollection >> activityHistogramFor: aProfileName in: aDataBaseFile [
|
||||
"I draw a histogram of the tweeter activity for a given profile name with data stored in aDataBaseFile.
|
||||
The database stores the individual tweets for this profile, with their type (tweet, retweet or reply),
|
||||
unique url and date.
|
||||
A proper schema of the data base still needs to be published.
|
||||
Is the one used in all references to aDataBaseFile."
|
||||
|
||||
| sample activityDataArray monthOfFirstTweet activityDataCollection histogramData plot |
|
||||
sample := TweetsCollection new.
|
||||
activityDataArray := sample monthlyActivityDataFor: aProfileName in: aDataBaseFile.
|
||||
(activityDataArray size > 0)
|
||||
ifFalse: [
|
||||
self inform: 'There is no data for ', aProfileName, ' in the database'
|
||||
]
|
||||
ifTrue:[
|
||||
"Detecting where happened the first tweet and storing only retweets over this value, will delete the
|
||||
outliers which correspond to retweets of original tweets far away of the period of the sample"
|
||||
monthOfFirstTweet := activityDataArray detect: [ :each | (each value at: 1) > 0].
|
||||
activityDataCollection := OrderedCollection new.
|
||||
activityDataArray do: [ :each |
|
||||
(each key >= monthOfFirstTweet key) ifTrue: [
|
||||
activityDataCollection add: {
|
||||
each key asString .
|
||||
each value at: 1 .
|
||||
each value at: 2 .
|
||||
each value at: 3 }
|
||||
]
|
||||
].
|
||||
|
||||
"This part was adapted from the awesome roassal examples"
|
||||
plot := RTGrapher new.
|
||||
histogramData := RTMultipleData new.
|
||||
histogramData barShape color: Color green.
|
||||
histogramData
|
||||
points: activityDataCollection;
|
||||
addMetric: #second;
|
||||
addMetric: #third;
|
||||
addMetric: #fourth.
|
||||
|
||||
"Horizontal text"
|
||||
"d barChartWithBarCenteredTitle: #first."
|
||||
|
||||
"Rotated text with integer axis"
|
||||
histogramData barChartUsing: (RTBarLabelFactory new label: #first; fontSize: 7).
|
||||
plot add: histogramData.
|
||||
plot axisY noDecimal.
|
||||
^ plot
|
||||
]
|
||||
]
|
||||
|
||||
{ #category : #'data queries' }
|
||||
TweetsCollection >> importTweetsFromJSONFile: aJSONFile [
|
||||
"I import all the tweets for aJSONFile and convert them in tweets inside a TweetCollection"
|
||||
| stream truncated jsonData currentTweet |
|
||||
stream := aJSONFile readStream.
|
||||
"We need to truncate the original file to quite the first line, which is the name of the exported array, so NeoJSONReader doesn't complain"
|
||||
truncated := WriteStream on: String new.
|
||||
stream contents lines allButFirstDo: [ :each | truncated nextPutAll: each ].
|
||||
|
||||
jsonData := NeoJSONReader fromString: truncated contents asString.
|
||||
jsonData do: [:each |
|
||||
currentTweet := Tweet new.
|
||||
currentTweet
|
||||
message: (each at: 'text');
|
||||
profile: ((each at: 'user') at: 'screen_name');
|
||||
date: ((each at: 'created_at') copyFrom: 1 to: 19) asDateAndTime.
|
||||
"Detecting the kind of message and processing accordingly"
|
||||
(each keys includes: 'in_reply_to_status_id')
|
||||
ifTrue: [
|
||||
currentTweet
|
||||
type: 'reply';
|
||||
url: '/', (currentTweet profile, '/status/', (each at: 'id_str'))]
|
||||
ifFalse: [(each keys includes: 'retweeted_status')
|
||||
ifTrue: [
|
||||
currentTweet
|
||||
type: 'retweet';
|
||||
url: '/',
|
||||
(((each at: 'retweeted_status') at: 'user') at: 'screen_name'),
|
||||
'/status/',
|
||||
((each at: 'retweeted_status') at: 'id_str').
|
||||
]
|
||||
ifFalse: [
|
||||
currentTweet
|
||||
type: 'tweet';
|
||||
url: '/', (currentTweet profile, '/status/', (each at: 'id_str')) ]
|
||||
].
|
||||
"Detecting hashtags"
|
||||
"(((each at: 'entities') at: 'hashtags') size > 0)
|
||||
ifTrue: [
|
||||
(each at: 'entities') at: 'hashtags'
|
||||
]."
|
||||
self tweets add: currentTweet.
|
||||
].
|
||||
]
|
||||
|
||||
{ #category : #'data queries' }
|
||||
TweetsCollection >> loadTweetsFor: aProfileName from: aDataBaseFile [
|
||||
"I select all the tweets for aProfileName in a given database"
|
||||
| db queryResults temporalTweet |
|
||||
|
||||
"openning connection"
|
||||
db := NBSQLite3Connection on: aDataBaseFile.
|
||||
db open.
|
||||
"Querying the data base"
|
||||
queryResults := (db execute: 'select * from tweets where profile="',aProfileName,'";') rows.
|
||||
db close.
|
||||
|
||||
queryResults do: [ :each |
|
||||
temporalTweet := Tweet new.
|
||||
temporalTweet
|
||||
url: (each at: 'url');
|
||||
date: (TimeStamp fromUnixTime: (each at: 'date')) asUTC;
|
||||
type: (each at: 'type');
|
||||
message: (each at: 'message');
|
||||
profile: (each at: 'profile').
|
||||
self tweets add: temporalTweet
|
||||
].
|
||||
]
|
||||
|
||||
{ #category : #'data queries' }
|
||||
TweetsCollection >> monthlyActivityDataFor: aProfileName in: aDataBaseFile [
|
||||
"I present a histogram of the tweets that differenciates tweets, retweets and replies,
|
||||
for a given profile in a given SQLite database (for the moment I supposse that the profile exist there
|
||||
and data base schema is correct)"
|
||||
| db queryResults firstMonth lastMonth currentMonth activityCalendar monthOfFirstTweet |
|
||||
|
||||
"openning connection"
|
||||
db := NBSQLite3Connection on: aDataBaseFile.
|
||||
db open.
|
||||
"Querying the data base"
|
||||
db execute: 'create temporary table profile_tweets as select * from tweets where profile="',aProfileName,'";'.
|
||||
queryResults := (db execute:
|
||||
'SELECT strftime("%Y-%m",datetime(date, "unixepoch","localtime")) as month, type, count(*) as amount
|
||||
FROM profile_tweets GROUP BY strftime("%Y-%m",datetime(date, "unixepoch","localtime")), type;') rows.
|
||||
db execute: 'drop table if exists profile_tweets;'.
|
||||
db close.
|
||||
activityCalendar := Dictionary new.
|
||||
(queryResults size > 0)
|
||||
ifFalse: [
|
||||
self inform: 'There is no data for that profile in the database'
|
||||
]
|
||||
ifTrue: [
|
||||
"Detecting where happened the first tweet and storing only retweets over this value, will delete the
|
||||
outliers which correspond to retweets of original tweets far away of the period of the sample"
|
||||
monthOfFirstTweet := queryResults detect: [ :each | (each at: 'month') notNil].
|
||||
firstMonth := ((monthOfFirstTweet at: 'month'), '-01') asDate asMonth.
|
||||
lastMonth := ((queryResults last at: 'month'), '-01') asDate asMonth.
|
||||
currentMonth := firstMonth.
|
||||
[ currentMonth = (lastMonth next)]
|
||||
whileFalse:[
|
||||
activityCalendar at: (currentMonth) put: { 0 . 0 . 0 }.
|
||||
currentMonth := currentMonth next.
|
||||
].
|
||||
queryResults do: [ :each |
|
||||
(each at: 'type') = 'tweet' & ((each at: 'month') notNil) ifTrue: [
|
||||
currentMonth := ((each at: 'month'), '-01') asDate asMonth.
|
||||
(activityCalendar at: currentMonth)
|
||||
at: 1
|
||||
put: (((activityCalendar at: currentMonth) at: 1) + (each at: 'amount'))
|
||||
].
|
||||
(each at: 'type') = 'retweet' ifTrue: [
|
||||
currentMonth := ((each at: 'month'), '-01') asDate asMonth.
|
||||
(activityCalendar at: currentMonth)
|
||||
at: 2
|
||||
put: (((activityCalendar at: currentMonth) at: 2) + (each at: 'amount'))
|
||||
].
|
||||
(each at: 'type') = 'reply' ifTrue: [
|
||||
currentMonth := ((each at: 'month'), '-01') asDate asMonth.
|
||||
(activityCalendar at: currentMonth)
|
||||
at: 3
|
||||
put: (((activityCalendar at: currentMonth) at: 3) + (each at: 'amount'))
|
||||
]
|
||||
].
|
||||
].
|
||||
^ (activityCalendar associations sorted)
|
||||
]
|
||||
|
||||
{ #category : #'data storage / persistance' }
|
||||
TweetsCollection >> populateDataBase: aDataBaseFile [
|
||||
"I populate a SQLite database file with my tweets data"
|
||||
| db |
|
||||
"openning connection"
|
||||
db := NBSQLite3Connection on: aDataBaseFile.
|
||||
db open.
|
||||
"Creating the data base tweets schema"
|
||||
db execute:
|
||||
'create table if not exists tweets (
|
||||
url text primary key,
|
||||
profile text,
|
||||
date integer,
|
||||
type text,
|
||||
message text
|
||||
);'.
|
||||
"Populating the database"
|
||||
self tweets do: [:each |
|
||||
db execute: 'INSERT INTO tweets values (?, ?, ?, ?, ?);'
|
||||
with: {
|
||||
each url .
|
||||
each profile .
|
||||
each date ifNotNil: [each date asUnixTime asString ].
|
||||
each type .
|
||||
each message }.
|
||||
].
|
||||
db close.
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
TweetsCollection >> ringOverview [
|
||||
"I present a overview of the tweets as a ring that differenciates tweets, retweets and replies"
|
||||
| totalTweets replies retweets ring |
|
||||
replies := 0.
|
||||
retweets := 0.
|
||||
tweets do: [ :each |
|
||||
(each type = 'reply') ifTrue: [replies := replies + 1].
|
||||
(each type = 'retweets') ifTrue: [retweets := retweets + 1]].
|
||||
totalTweets := (self tweets size) - replies - retweets.
|
||||
ring := RTPieBuilder new.
|
||||
ring interaction popup.
|
||||
ring shape current
|
||||
innerRadius: 80;
|
||||
externalRadius: 100.
|
||||
ring objects: {totalTweets . retweets . replies}.
|
||||
(ring slice: #value)ifNotNil: [ :group |
|
||||
group do: [:each | each @ (RTDraggable groupToDrag: group)]
|
||||
].
|
||||
ring normalizer distinctColor.
|
||||
ring build.
|
||||
^ ring.
|
||||
]
|
||||
|
||||
{ #category : #'data visualization' }
|
||||
TweetsCollection >> ringOverviewFor: aProfileName in: aDataBaseFile [
|
||||
"I present a overview of the tweets as a ring that differenciates tweets, retweets and replies,
|
||||
for a given profile in a given SQLite database (for the moment I supposse that the profile exist there
|
||||
and data base schema is correct)"
|
||||
| db totalTweets replies retweets ring |
|
||||
|
||||
"openning connection"
|
||||
db := NBSQLite3Connection on: aDataBaseFile.
|
||||
db open.
|
||||
"Querying the data base"
|
||||
retweets := (db execute: 'select * from tweets where profile="', aProfileName ,'" and type="retweet";') rows size.
|
||||
replies := (db execute: 'select * from tweets where profile="', aProfileName ,'" and type="reply";') rows size.
|
||||
totalTweets := (db execute: 'select * from tweets where profile="', aProfileName ,'";') rows size - retweets - replies.
|
||||
db close.
|
||||
(totalTweets > 0)
|
||||
ifFalse: [
|
||||
self inform: 'There are no tweets in the database for that profile'
|
||||
]
|
||||
ifTrue: [
|
||||
ring := RTPieBuilder new.
|
||||
ring interaction popup.
|
||||
ring shape current
|
||||
innerRadius: 80;
|
||||
externalRadius: 100.
|
||||
ring objects: {totalTweets . retweets . replies}.
|
||||
(ring slice: #value)ifNotNil: [ :group |
|
||||
group do: [:each | each @ (RTDraggable groupToDrag: group)]
|
||||
].
|
||||
ring normalizer distinctColor.
|
||||
ring build.
|
||||
^ ring].
|
||||
]
|
||||
|
||||
{ #category : #'data scrapping' }
|
||||
TweetsCollection >> scrapTweetsFromHtmlFile: aHtmlFile [
|
||||
"I scraps tweets from a downloaded html file.
|
||||
On how to download such file for any given public twitter profile look at:
|
||||
http://blog.databigbang.com/scraping-web-sites-which-dynamically-load-data/
|
||||
"
|
||||
| tweetsDump htmlTree tweetsHtml tweet unixTime answersArray tweetsTemp profile |
|
||||
|
||||
tweetsDump := aHtmlFile readStream.
|
||||
htmlTree := Soup fromString: tweetsDump contents.
|
||||
profile := (((htmlTree findAllTagsByClass: 'ProfileHeaderCard-screennameLink') at: 1) attributeAt: 'href') copyReplaceAll: '/' with: ''.
|
||||
tweetsHtml := htmlTree findAllTagsByClass: 'tweet'.
|
||||
tweetsTemp := OrderedCollection new.
|
||||
tweetsHtml allButLastDo: [:each |
|
||||
tweet := Tweet new.
|
||||
(each findAllTagsByClass: '_timestamp') size > 0
|
||||
ifTrue: [
|
||||
unixTime := (((each findAllTagsByClass: '_timestamp') at: 1) attributeAt: 'data-time') asInteger.
|
||||
tweet date: (TimeStamp fromUnixTime: unixTime) asUTC
|
||||
].
|
||||
answersArray := each findAllTagsByClass: 'js-retweet-text'.
|
||||
(answersArray size = 1)
|
||||
ifTrue: [tweet type: 'retweet']
|
||||
ifFalse: [
|
||||
(each attributeAt: 'data-is-reply-to') isString
|
||||
ifTrue: [tweet type: 'reply']
|
||||
ifFalse: [tweet type: 'tweet']
|
||||
].
|
||||
tweet url: (each attributeAt: 'data-permalink-path').
|
||||
(each findAllTagsByClass: 'TweetTextSize') size > 0
|
||||
ifTrue: [tweet message: (((each findAllTagsByClass: 'TweetTextSize') at: 1) text)].
|
||||
tweet profile: profile.
|
||||
tweetsTemp add: tweet.
|
||||
].
|
||||
self tweets: tweetsTemp.
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
TweetsCollection >> tweets [
|
||||
^ tweets ifNil: [tweets := OrderedCollection new]
|
||||
]
|
||||
|
||||
{ #category : #accessing }
|
||||
TweetsCollection >> tweets: anOrderedCollection [
|
||||
tweets := anOrderedCollection
|
||||
]
|
||||
|
|
@ -162,6 +162,26 @@ TwitterProfile >> lastTweets: anObject [
|
|||
lastTweets := anObject
|
||||
]
|
||||
|
||||
{ #category : #'data storage / persistance' }
|
||||
TwitterProfile >> loadDataFor: aProfileName fromDatabase: aDataBaseFile [
|
||||
| db queryResults |
|
||||
|
||||
"openning connection"
|
||||
db := NBSQLite3Connection on: aDataBaseFile.
|
||||
db open.
|
||||
"Querying the data base"
|
||||
queryResults := (db execute: 'SELECT * FROM profiles WHERE screenName="',aProfileName,'";') rows at: 1.
|
||||
db close.
|
||||
self
|
||||
screenName: (queryResults at: 'screenName');
|
||||
name: (queryResults at: 'name');
|
||||
avatar: (queryResults at: 'avatar');
|
||||
bio: (queryResults at: 'bio');
|
||||
favs: (queryResults at: 'favs');
|
||||
followers: (queryResults at: 'followers');
|
||||
following: (queryResults at: 'following')
|
||||
]
|
||||
|
||||
{ #category : #'data storage / persistance' }
|
||||
TwitterProfile >> loadDataFromFile: aFileReference [
|
||||
"Opens the twitter profile from aFileReference stored in the STON format"
|
||||
|
|
@ -226,6 +246,39 @@ TwitterProfile >> name: anObject [
|
|||
name := anObject
|
||||
]
|
||||
|
||||
{ #category : #'data storage / persistance' }
|
||||
TwitterProfile >> populateDataBase: aDataBaseFile [
|
||||
"I populate a SQLite database file with myself data"
|
||||
| db |
|
||||
"openning connection"
|
||||
db := NBSQLite3Connection on: aDataBaseFile.
|
||||
db open.
|
||||
"Creating the data base tweets schema"
|
||||
db execute:
|
||||
'create table if not exists profiles (
|
||||
screenName text primary key,
|
||||
name text,
|
||||
avatar blob,
|
||||
bio text,
|
||||
favs integer,
|
||||
followers integer,
|
||||
following integer,
|
||||
location text
|
||||
);'.
|
||||
"Populating the database"
|
||||
db execute: 'INSERT INTO profiles values (?, ?, ?, ?, ?, ?, ?, ?);'
|
||||
with: {
|
||||
self screenName.
|
||||
self name.
|
||||
self avatar.
|
||||
self bio.
|
||||
self favs.
|
||||
self followers.
|
||||
self following.
|
||||
self location}.
|
||||
db close.
|
||||
]
|
||||
|
||||
{ #category : #'data storage / persistance' }
|
||||
TwitterProfile >> saveToFile: aFileReference [
|
||||
"Saves the twitter profile to aFileReference in the STON format"
|
||||
|
|
@ -235,6 +288,24 @@ TwitterProfile >> saveToFile: aFileReference [
|
|||
stream nextPutAll: (STON toStringPretty: self).
|
||||
]
|
||||
|
||||
{ #category : #dataweek }
|
||||
TwitterProfile >> sayBye [
|
||||
"Just says hello to all the people which is listening. A dummy example on how to create new messages"
|
||||
Transcript open.
|
||||
Transcript show: 'Adios, perfil de Twitter, despidiéndose. Pásala bueno ;-)'
|
||||
|
||||
|
||||
]
|
||||
|
||||
{ #category : #dataweek }
|
||||
TwitterProfile >> sayHello [
|
||||
"Just says hello to all the people which is listening. A dummy example on how to create new messages"
|
||||
Transcript open.
|
||||
Transcript show: 'Hola! soy un perfil de Twitter :-)'
|
||||
|
||||
|
||||
]
|
||||
|
||||
{ #category : #'data scrapping' }
|
||||
TwitterProfile >> scrapAvatarFrom: aHtmlString [
|
||||
"Finds the avatar in a twitter's main page profile, scales it (200x200), cast it agains different formats (jpeg, png) and returns it"
|
||||
|
|
@ -245,15 +316,14 @@ TwitterProfile >> scrapAvatarFrom: aHtmlString [
|
|||
(avatarUrl asLowercase endsWith: '.png')
|
||||
ifTrue: [avatarImage := ZnEasy getPng: avatarUrl].
|
||||
((avatarUrl asLowercase endsWith: '.jpeg') or: (avatarUrl asLowercase endsWith: '.jpg'))
|
||||
ifTrue: [
|
||||
avatarImage := ZnEasy get: avatarUrl.
|
||||
"(PNGReadWriter on: avatarImage ) nextPutImage: (JPEGReadWriter on: avatarImage )" ].
|
||||
ifTrue: [avatarImage := ZnEasy getJpeg: avatarUrl].
|
||||
^avatarImage.
|
||||
]
|
||||
|
||||
{ #category : #'data scrapping' }
|
||||
TwitterProfile >> scrapDataFromProfile: aProfileName [
|
||||
"Scraps data from aProfileName and fills out the TwitterProfile. The profile name is the last part of a twitter profile url
|
||||
TwitterProfile >> scrapDataForProfile: aProfileName [
|
||||
"Scraps data from aProfileName and fills out the TwitterProfile.
|
||||
The profile name is the last part of a twitter profile url
|
||||
(i.e: 'https://twitter.com/aProfileName')."
|
||||
|
||||
| client source numericalData anUrl |
|
||||
|
|
@ -282,6 +352,24 @@ TwitterProfile >> scrapDataFromProfile: aProfileName [
|
|||
|
||||
]
|
||||
|
||||
{ #category : #'data scrapping' }
|
||||
TwitterProfile >> scrapFollowersForProfile: aProfileName [
|
||||
"Scraps data from a predefined profile name"
|
||||
|
||||
| client source numericalData anUrl |
|
||||
anUrl := 'https://twitter.com/', aProfileName.
|
||||
client := ZnClient new.
|
||||
client get: anUrl.
|
||||
client isSuccess
|
||||
ifTrue:[
|
||||
source := Soup fromString: (client) contents asString.
|
||||
numericalData := (source findAllTagsByClass: 'ProfileNav-value') collect:[:each | each text].
|
||||
followers := self asNumber: (numericalData at: 3).
|
||||
]
|
||||
ifFalse:[self inform: 'Algo salió mal. Verifique su conexión a Internet y que el contenido buscado estén disponibles'].
|
||||
^ self followers
|
||||
]
|
||||
|
||||
{ #category : #'data scrapping' }
|
||||
TwitterProfile >> scrapTweetsFromFile: aFile [
|
||||
"Scraps tweets data from aFile, wich contains tweets scrapped from a public profile."
|
||||
|
|
|
|||
Loading…
Reference in New Issue