From f2ccfd15ccd6ac111d996b4e7d834527b12d7684 Mon Sep 17 00:00:00 2001 From: akevalion Date: Thu, 7 Apr 2016 16:35:40 +0000 Subject: [PATCH] Updae Dataviz to merging with Roassal2-RTSunburst Co-authored-by: yanekgil --- src/Dataviz/ManifestDataviz.class.st | 18 + src/Dataviz/OpenSpending.class.st | 64 +++ src/Dataviz/PublishedMedInfo.class.st | 635 ++++++++++++++++++++++++++ src/Dataviz/Tweet.class.st | 314 ++++++++++++- src/Dataviz/TweetsCollection.class.st | 345 ++++++++++++++ src/Dataviz/TwitterProfile.class.st | 98 +++- 6 files changed, 1445 insertions(+), 29 deletions(-) create mode 100644 src/Dataviz/ManifestDataviz.class.st create mode 100644 src/Dataviz/OpenSpending.class.st create mode 100644 src/Dataviz/PublishedMedInfo.class.st create mode 100644 src/Dataviz/TweetsCollection.class.st diff --git a/src/Dataviz/ManifestDataviz.class.st b/src/Dataviz/ManifestDataviz.class.st new file mode 100644 index 0000000..1c148a6 --- /dev/null +++ b/src/Dataviz/ManifestDataviz.class.st @@ -0,0 +1,18 @@ +" +I store metadata for this package. These meta data are used by other tools such as the SmalllintManifestChecker and the critics Browser +" +Class { + #name : #ManifestDataviz, + #superclass : #PackageManifest, + #category : #Dataviz +} + +{ #category : #'code-critics' } +ManifestDataviz class >> ruleRBSentNotImplementedRuleV1FalsePositive [ + ^ #(#(#(#RGClassDefinition #(#PublishedMedInfo)) #'2015-12-06T12:21:46.001583-05:00') ) +] + +{ #category : #'code-critics' } +ManifestDataviz class >> ruleRBUnclassifiedMethodsRuleV1FalsePositive [ + ^ #(#(#(#RGClassDefinition #(#MedAgency)) #'2015-12-17T23:52:38.455558-05:00') ) +] diff --git a/src/Dataviz/OpenSpending.class.st b/src/Dataviz/OpenSpending.class.st new file mode 100644 index 0000000..48f45c7 --- /dev/null +++ b/src/Dataviz/OpenSpending.class.st @@ -0,0 +1,64 @@ +" +I'm used to help in the citizen oversight of public spending. +I load scrapped information of public Government sites (starting with +Colombian ones) and help in its understanding by providing visualizations and other techniques. +" +Class { + #name : #OpenSpending, + #superclass : #Object, + #instVars : [ + 'contractsData' + ], + #category : #'Dataviz-OpenSpending' +} + +{ #category : #accessing } +OpenSpending >> contractsData [ + ^ contractsData +] + +{ #category : #accessing } +OpenSpending >> contractsData: anObject [ + contractsData := anObject +] + +{ #category : #'data import' } +OpenSpending >> listSectors [ + "I import data from aFile storaged in Comma Separated Values (CSV) format. + Is supposed that they're taken from a query that contains ths contracting sectors + according to https://en.wikipedia.org/wiki/UNSPSC" + | queryResults dbSectorsClassifier | + dbSectorsClassifier := FileLocator documents / 'DataWeek' / 'clasificador.csv'. + queryResults := (RTTabTable new input: dbSectorsClassifier contents usingDelimiter: $,) values. + "table removeFirstRow. + self contractsData: table." + ^ queryResults +] + +{ #category : #'data import' } +OpenSpending >> loadDataFromCSV: aFile usingDelimiter: aCharacter [ + "I import data from aFile storaged in Comma Separated Values (CSV) format. + Is supposed that they're taken from a query that sums the sectors of the economy" + | table | + table := RTTabTable new input: aFile contents usingDelimiter: aCharacter. + table removeFirstRow. + self contractsData: table. +] + +{ #category : #'as yet unclassified' } +OpenSpending >> showTreeMap [ + "I show a treemap visualization of my contract data" + + | builder popup sectors subtotals | + popup := RTPopup new. + sectors := self contractsData valuesOfColumn: 1. + subtotals := self contractsData valuesOfColumn: 2. + popup text: [:index | (sectors at: index), String cr, (subtotals at: index) ]. + builder := RTTreeMapBuilder new. + builder shape color: [:n | Color random]. + builder from: (1 to: (self contractsData numberOfRows)) using: [#()]. + builder weight: [:n | (self contractsData valuesOfColumn: 2) at: n ]. + builder build. + builder view elements @ popup. + ^ builder view. +] diff --git a/src/Dataviz/PublishedMedInfo.class.st b/src/Dataviz/PublishedMedInfo.class.st new file mode 100644 index 0000000..3aff0e4 --- /dev/null +++ b/src/Dataviz/PublishedMedInfo.class.st @@ -0,0 +1,635 @@ +" +Please comment me using the following template inspired by Class Responsibility Collaborator (CRC) design: + +For the Class part: State a one line summary. For example, ""I represent a paragraph of text"". +I represent the published medicine information from several sources for +a particular medicine. + +My main responsabilities are: +- Visualize published medicine information to help in comparing several +agencies. +- I know how much information is published for a particular medicine. + +My main collaborator is: +- MedDataItem: I store several of them. + +" +Class { + #name : #PublishedMedInfo, + #superclass : #Object, + #instVars : [ + 'medName', + 'medDataItems', + 'medDataKeys', + 'medDataMatrix', + 'adminDataSize', + 'arcWidth', + 'columnLabels' + ], + #category : #'Dataviz-MedicineInfo' +} + +{ #category : #utility } +PublishedMedInfo >> addLabelsFrom: anArray surrounding: aView withRadio: aDistance [ + "I put a list of labels on aView which around a circle with a given radio" + | angle | + + 1 to: anArray size do: [ :index | |label| + angle := (index * 360/ anArray size) degreesToRadians. + label := RTLabel new + text: (anArray at: index) asString; + height: aDistance / 30. + aView + add: (label element translateBy: (Point r: (aDistance)* 1.06 theta: angle - 0.075)) + ]. + ^ aView. +] + +{ #category : #utility } +PublishedMedInfo >> addLegendTo: aView titled: aString withData: anArray withColors: aColorPalette [ + "Adds a legend with a given title to a given view" + + | legend | + legend := RTLegendBuilder new. + legend addText: aString. + + aView + add: legend. +] + +{ #category : #utility } +PublishedMedInfo >> addLineSeparatorsTo: aView withData: data columnsDistance: aDistance centerSized: internalRadio ringSized: ringSize [ + | e1 e2 ang | + 1 to: data children size do: [ :i | + Transcript show: i. + e1 := (RTBox new size: 1) element. + ang := (i * 360 / data children size) degreesToRadians. + e1 translateTo: (internalRadio * ang cos)@(internalRadio * ang sin). + e2 := (RTBox new size: 1) element. + e2 translateTo: ((internalRadio + ringSize) * ang cos )@ ((internalRadio + ringSize)* ang sin). + aView + add: e1; + add: e2; + add: (RTLine new width: aDistance - 4 ; color: Color veryLightGray; edgeFrom: e1 to: e2) + ] + +] + +{ #category : #utility } +PublishedMedInfo >> addRotatedLabelsFrom: anArray surrounding: aView withRadio: aDistance withAngularCorrection: aValue [ + "I put a list of rotated labels on aView which around a circle with a given radio" + | angle | + + 1 to: anArray size do: [ :index | |label| + angle := ((index * 360/ anArray size) - aValue) negated degreesToRadians. + label := (RTRotatedLabel new + text: (anArray at: index) asString; + height: aDistance / 22) element. + label translateBy: (Point r: label trachelShape notRotatedWidth/2+(aDistance)*1.06 theta: angle). + aView add: label. + angle := angle radiansToDegrees. + angle := angle + ((angle between: -270 and: -90) ifTrue: [ 180 ] ifFalse: [ 0 ]). + label trachelShape angleInDegree: angle. + ]. + ^ aView. +] + +{ #category : #accessing } +PublishedMedInfo >> adminDataSize [ + "I calculate the amount of administrative data in the matrix. By convention all administrative data starts with '*' in its name" + + ^ adminDataSize isNil + ifTrue: [ + (self medDataKeys select: [:key | (key at: 1) = $*]) size + ] +] + +{ #category : #accessing } +PublishedMedInfo >> adminDataSize: anObject [ + adminDataSize := anObject +] + +{ #category : #utility } +PublishedMedInfo >> arcWidth [ + ^ arcWidth +] + +{ #category : #accessing } +PublishedMedInfo >> arcWidth: anObject [ + arcWidth := anObject +] + +{ #category : #utility } +PublishedMedInfo >> colorForWithoutBiosimilar [ + "Returns a customized color for a medicament which has not biosimilar in its published info" + + ^ Color r: 218/255 g: 223/255 b: 225/255 +] + +{ #category : #utility } +PublishedMedInfo >> colorPalette16 [ + "Returns a custom color palette for administrative data. In the data matrix, this correspond + to the rows which name starts by '*' (first two admin data, contry and variant are already mapped in the name of each arc and don't + start with '*'" + + | baseColors newColorPalette index | + baseColors := RTColorPalette qualitative + colors: 12 + scheme:'Set3'. + index := 1. + newColorPalette := Array new: 16. + baseColors do: [ :color | + newColorPalette at: index put: color. + index := index + 1. + ]. + "Customizing some individual colors to get a more constrasted palette, useful to distinguis in + large information collections" + newColorPalette + at: 3 put: (Color r: 71/255 g:30/255 b: 253/255); + at: 8 put: (Color r: 134/255 g:43/255 b: 80/255); + at: 9 put: (Color r: 156/255 g:143/255 b: 50/255); + at: 11 put: (Color r: 164/255 g:44/255 b: 107/255); + at: 13 put: (Color r:0.75 g:0.22 b:0.17); + at: 14 put: (Color r:0.42 g:0.48 b:0.54); + at: 15 put: (Color r:0.83 g:0.33 b:0); + at: 16 put: (Color r: 211/255 g:26/255 b: 239/255). + ^ newColorPalette. +] + +{ #category : #utility } +PublishedMedInfo >> colorPalette26 [ + "Returns a custom color palette for prescrition and use data. In the data matrix, this correspond + to the rows which name doesn't start by '*' (first two admin data, contry and variant are already mapped in the name of each arc and doesn't + start with '*'" + + | baseColors newColorPalette index | + baseColors := RTColorPalette qualitative + colors: 20 + scheme:'FlatUI1'. + index := 1. + newColorPalette := Array new: 26. + baseColors do: [ :color | + newColorPalette at: index put: color. + index := index + 1. + ]. + newColorPalette + at: 2 put: (Color r:0.65 g:0.22 b:0.23); + at: 4 put: (Color r:1 g:0.72 b:0.1); + at: 6 put: (Color r:255/255 g:230/255 b:141/255); + at: 8 put: (Color r:178/255 g:24/255 b:17/255); + at: 14 put: (Color r:255/255 g:214/255 b:208/255); + at: 18 put: (Color r:218/255 g:0/255 b:132/255); + at: 21 put: (Color r:223/255 g:93/255 b:255/255); + at: 22 put: (Color r:0.29 g:0.47 b:0.75); + at: 23 put: (Color r:0.64 g:0.87 b:0.82); + at: 24 put: (Color r:131/255 g:178/255 b:10/255); + at: 25 put: (Color r:0.91 g:0.83 b:038); + at: 26 put: (Color r:0.42 g:0.48 b:054). + ^ newColorPalette. +] + +{ #category : #accessing } +PublishedMedInfo >> columnLabels [ + ^ columnLabels +] + +{ #category : #accessing } +PublishedMedInfo >> columnLabels: anObject [ + columnLabels := anObject +] + +{ #category : #'data visualization' } +PublishedMedInfo >> exploreDataForCountriesFromRow: firstRowIndex upTo: lastRowIndex coloredWith: aColorPalette centerTitled: aString arcWidth: aNumber [ + "I show a subset of the matrix rows using a sunburst visualization variation" + + self arcWidth: aNumber. + ^ self + exploreMatrix: (self medDataMatrix copyFrom: firstRowIndex to: lastRowIndex) + by: 'country' + coloredWith: aColorPalette + centerSized: 200 + separationSized: 5 + slicedFrom: firstRowIndex + centerTitled: aString +] + +{ #category : #'data visualization' } +PublishedMedInfo >> exploreMatrix: aMatrix by: type coloredWith: aColorPalette centerSized: centerSize separationSized: separationSize slicedFrom: anInteger centerTitled: aString [ + "I create a sunburst diagram for matrix alike information using aPalette for each of the rings on the sunburst. + Information of the popup is taken according to the type of exploration that is being done in the matrix, which can + done by the country which is publishing the info or by properties that are bein published. + This code has a lot of parameters and maybe needs serious refactoring" + + | b data externalCircle internalCircle medLabel index | + + data := self matrixToTreeReversed: aMatrix. + b := RTSunburstBuilder new. + b strategy + centerWidth: centerSize; + hasCenter: false; + arcWidth: self arcWidth; + radialSpacing: 50. + internalCircle := RTEllipse new + width: centerSize*2; height: centerSize*2; + color: Color white trans; + borderColor: Color veryLightGray; + borderWidth: 1. + externalCircle := RTEllipse new + width: internalCircle element width + (2 * aMatrix size * (self arcWidth + 1)); + height: internalCircle element width + (2 * aMatrix size * (self arcWidth + 1)); + color: Color white trans; + borderColor: Color veryLightGray; + borderWidth: 1. + b view add: externalCircle element. + b view add: internalCircle element. + medLabel := RTLabel new + text: aString, String cr, self medName; + color: Color black; + height: (internalCircle element width)/20 asInteger. + b view add: (medLabel element translateBy: (internalCircle element center)). + b radialSpacing: 1. + index := 1. + b shape color: [ :node | + node header asString size > 1 + ifTrue: [ Color black ] + ifFalse: [ + node header asInteger isZero + ifTrue: [ Color white ] + ifFalse: [ node header asInteger = 2 + ifTrue: [ self colorForWithoutBiosimilar ] + ifFalse: [ aColorPalette at: node level ] + ] + ] + ]. + b interaction noInteractions. + type asLowercase = 'country' ifTrue: [ + b interaction + addInteraction: + (RTPopup text: [:d | (self medDataKeys at: ((aMatrix size) - d level + anInteger + 1)), '-> ', d header]). + ]. + type asLowercase = 'property' ifTrue: [ + b interactions + add: + (RTPopup text: [:d | self labelsForCountries at: (d level - 1)]). + ]. + b explore: data using: #children. + b strategy hasCenter: false. + b build. + b view @ RTZoomableBoxView. + self + addLineSeparatorsTo: b view + withData: data + columnsDistance: separationSize + centerSized: (internalCircle element width / 2) + ringSized: ((externalCircle element width/2) - (internalCircle element width/2)). + ^ {'theView' -> b view . 'externalRadio' -> (externalCircle element width/2)} asDictionary. +] + +{ #category : #utility } +PublishedMedInfo >> labelsForAdminProperties [ + "I got the information about the administrative properties that are being published by + all countries public health agencies. + This information is a subset of medDataKeys" + + ^ self medDataKeys copyFrom: 3 to: self adminDataSize + 2. +] + +{ #category : #utility } +PublishedMedInfo >> labelsForCountries [ + "I got the information about the country who is publishing the medicine info. + This information is stored in the first row of the medDataMatrix. + Countries use the ISO 3166-1 alpha 3 standard." + + | labels allCountries | + allCountries := (self medDataMatrix at: 1). + labels := OrderedCollection new. + 1 to: allCountries size by: 2 do: [:index | + labels add: (allCountries at: index). + ]. + ^ labels reversed. +] + +{ #category : #utility } +PublishedMedInfo >> labelsForCountriesAndVariants [ + "I got the information about the country and variant of medicine info that is being published by a + country public health agency. + This information is stored in the first two rows of the medDataMatrix. + Countries use the ISO 3166-1 alpha 3 standard and for the medicine variant we use complete words + in the CVS but select only the first letter for visualization purposes, so the first letter is + supposed to be different on each variant." + + | labels countries variants | + countries := self medDataMatrix at: 1. + variants := self medDataMatrix at: 2. + labels := Array new: countries size. + 1 to: countries size do: [:index | + labels + at: index + put: (countries at: (countries size + 1 - index)), + '-', + (((variants at: (variants size + 1 - index)) copyFrom: 1 to: 1) asLowercase) + ]. + ^ labels +] + +{ #category : #utility } +PublishedMedInfo >> labelsForPUProperties [ + "I got the information about the properties that are being published by all countries public health agencies. + This information is a subset of medDataKeys" + + ^ self medDataKeys copyFrom: 3 + self adminDataSize to: self medDataKeys size. +] + +{ #category : #'data import' } +PublishedMedInfo >> loadDataFromCSV: aFile usingDelimiter: aCharacter [ + "I import data from aFile storaged in Comma Separated Values (CSV) format. + Is supposed that: + - First row contains medicine name information. + - First column contains the attributes that are being evaluated + - Other columns contain the result of such evaluation as numerical values" + | table | + table := RTTabTable new input: aFile contents usingDelimiter: aCharacter. + self medName: ((table valuesOfColumn: 2) at: 1). + table removeFirstRow. + self medDataKeys: table firstColumn. + table removeColumn: 1. + self medDataMatrix: table values. + + +] + +{ #category : #utility } +PublishedMedInfo >> matrixDataForAdmin [ + "I extract all the administrative information from the medDataMatrix as a transposed matrix, so it can be used + as an input for the matrixSunburst visualizations. + By convention, this data is stored after the admin data for each drug has ended" + + | submatrixForAdmin numberOfColums transposedMatrix transposedRow | + submatrixForAdmin := self medDataMatrix copyFrom: 3 to: (self adminDataSize + 2). + numberOfColums := (submatrixForAdmin at: 1) size. + transposedMatrix := OrderedCollection new. + 1 to: (numberOfColums) by: 2 do: [ :columIndex | + transposedRow := Array new: submatrixForAdmin size. + 1 to: submatrixForAdmin size do: [:index | + transposedRow at: index put: ((submatrixForAdmin at: index) at: columIndex) + ]. + transposedMatrix add: transposedRow + ]. + ^ transposedMatrix asArray +] + +{ #category : #utility } +PublishedMedInfo >> matrixDataForPU [ + "I extract all the prescription and use data from the medDataMatrix as a transposed matrix, so it can be used + as an input for the matrixSunburst visualizations. + By convention, this data is stored after the admin data for each drug has ended" + + | submatrixForPU numberOfColums transposedMatrix transposedRow | + submatrixForPU := self medDataMatrix copyFrom: 3 + self adminDataSize to: self medDataMatrix size. + numberOfColums := (submatrixForPU at: 1) size. + transposedMatrix := OrderedCollection new. + 1 to: (numberOfColums) by: 2 do: [ :columIndex | + transposedRow := Array new: submatrixForPU size. + 1 to: submatrixForPU size do: [:index | + transposedRow at: index put: ((submatrixForPU at: index) at: columIndex) + ]. + transposedMatrix add: transposedRow + ]. + ^ transposedMatrix asArray +] + +{ #category : #'data visualization' } +PublishedMedInfo >> matrixSunburstForAdminDataByCountry [ + "Shows a matrix sunburst for the drug administrative data in several public drug agencies, using the + properties of the medicine as tows and the countries that release the info (or not) as columns. + By convention, this data starts at the 3rd row and ends in 2 + self adminDataSize." + + | temp view radio | + temp := self + exploreDataForCountriesFromRow: 3 + upTo: 1 + self adminDataSize + coloredWith: self colorPalette16 + centerTitled: '' + arcWidth: 15. + view := temp at: 'theView'. + radio := temp at: 'externalRadio'. + ^ self + addLabelsFrom: self labelsForCountriesAndVariants + surrounding: view + withRadio: radio*1.05. +] + +{ #category : #'data visualization' } +PublishedMedInfo >> matrixSunburstForAdminDataByProperty [ + "Shows a matrix sunburst for the drug administrative data in several public drug agencies, using the + properties of the medicine as columns and the countries as rows." + + | temp view radio | + temp := self + exploreMatrix: self matrixDataForAdmin + by: 'property' + coloredWith: self colorPalette16 + centerSized: 200 + separationSized: 6 + slicedFrom: nil + centerTitled: ''. + view := temp at: 'theView'. + radio := temp at: 'externalRadio'. + ^ self + addRotatedLabelsFrom: self labelsForAdminProperties + surrounding: view + withRadio: radio * 1.05 + withAngularCorrection: 14.5 +] + +{ #category : #'data visualization' } +PublishedMedInfo >> matrixSunburstForAdminDataWithColorConvention [ + "Shows a matrix sunburst for the drug administrative data in several public drug agencies" + + | composer sunburst | + composer := RTComposer new. + sunburst := self matrixSunburstForAdminDataByCountry. + composer group: #sunburst. + composer view. +] + +{ #category : #'data visualization' } +PublishedMedInfo >> matrixSunburstForPUDataByCountry [ + "Shows a matrix sunburst for the drug prescripton and use data in several public drug agencies. + For convention, this data stats at the row 3 + self adminDataSize and ends at the last one of the matrix" + | temp view radio | + temp:= self + exploreDataForCountriesFromRow: 3 + self adminDataSize + upTo: self medDataKeys size + coloredWith: self colorPalette26 + centerTitled: '' + arcWidth: 15. + view := temp at: 'theView'. + radio := temp at: 'externalRadio'. + ^ self + addLabelsFrom: self labelsForCountriesAndVariants + surrounding: view + withRadio: radio*1.05. +] + +{ #category : #'data visualization' } +PublishedMedInfo >> matrixSunburstForPUDataByProperty [ + "Shows a matrix sunburst for the drug administrative data in several public drug agencies, using the + properties of the medicine as columns and the countries as rows." + + | temp view radio | + temp := self + exploreMatrix: self matrixDataForPU + by: 'property' + coloredWith: self colorPalette16 + centerSized: 200 + separationSized: 6 + slicedFrom: nil + centerTitled: ''. + view := temp at: 'theView'. + radio := temp at: 'externalRadio'. + ^ self + addRotatedLabelsFrom: self labelsForPUProperties + surrounding: view + withRadio: radio * 1.05 + withAngularCorrection: 9 +] + +{ #category : #utility } +PublishedMedInfo >> matrixToTree: aMatrix [ + "Converts aMatrix to a tree so it can be visualized using a sunburst" + + | currentNode tree column | + tree := GrafoscopioNode new level: 1. + (aMatrix at: 1) do: [ :label | + tree addNode: (GrafoscopioNode new header: label) + ]. + column := 1. + tree children do: [ :child | + currentNode := child. + 2 to: (aMatrix size) do: [:row | + currentNode addNode: (GrafoscopioNode new header: ((aMatrix at: row) at: column)). + currentNode := currentNode children at: 1. + ]. + column := column + 1. + ]. + ^ tree. +] + +{ #category : #utility } +PublishedMedInfo >> matrixToTreeReversed: aMatrix [ + "Converts aMatrix to a tree so it can be visualized using a sunburst. + Root of the tree is near to the last row, so the most exterior rings correspond to first rows, + and inner most ones correspond to the last rows" + + | currentNode tree column | + tree := GrafoscopioNode new level: 1. + (aMatrix at: aMatrix size) do: [ :label | + tree addNode: (GrafoscopioNode new header: label) + ]. + column := 1. + tree children do: [ :child | + currentNode := child. + (aMatrix size -1 ) to: 1 by: -1 do: [:row | + currentNode addNode: (GrafoscopioNode new header: ((aMatrix at: row) at: column)). + currentNode := currentNode children at: 1. + ]. + column := column + 1. + ]. + ^ tree. +] + +{ #category : #accessing } +PublishedMedInfo >> medDataItems [ + ^ medDataItems +] + +{ #category : #accessing } +PublishedMedInfo >> medDataItems: anObject [ + medDataItems := anObject +] + +{ #category : #accessing } +PublishedMedInfo >> medDataKeys [ + ^ medDataKeys +] + +{ #category : #accessing } +PublishedMedInfo >> medDataKeys: anObject [ + medDataKeys := anObject +] + +{ #category : #accessing } +PublishedMedInfo >> medDataMatrix [ + ^ medDataMatrix +] + +{ #category : #accessing } +PublishedMedInfo >> medDataMatrix: anObject [ + medDataMatrix := anObject +] + +{ #category : #accessing } +PublishedMedInfo >> medName [ + ^ medName +] + +{ #category : #accessing } +PublishedMedInfo >> medName: anObject [ + medName := anObject +] + +{ #category : #'data visualization' } +PublishedMedInfo >> showAdminDataColorConventions [ + "I create a legend showing the color palette and properties used in the matrix sunburst for administrative drug data" + + | legendBox adminDataKeys | + + legendBox := RTLegendBuilder new. + legendBox addText: 'Convenciones' asUppercase. + adminDataKeys := self medDataKeys copyFrom: 3 to: self adminDataSize + 1. + 1 to: adminDataKeys size do: [ :i | + legendBox addColor: (self colorPalette16 at: (self colorPalette16 size + 1 - i)) text: ((adminDataKeys at: i) copyReplaceAll: '* ' with: ''). + ]. + legendBox addColor: self colorForWithoutBiosimilar text: 'Sin biosimilar aprobado'. + legendBox addColor: Color white text: 'Sin informacion publicada'. + legendBox build. + ^ legendBox view +] + +{ #category : #'data visualization' } +PublishedMedInfo >> showCountryColorConventions [ + "I create a legend showing the color palette and countries used in the matrix sunburst for prescription drug data" + + | legendBox | + + legendBox := RTLegendBuilder new. + legendBox addText: 'Convenciones' asUppercase. + self labelsForCountries size to: 1 by: -1 do: [ :i | + legendBox addColor: (self colorPalette16 at: i + 1) text: (self labelsForCountries at: i). + ]. + legendBox addColor: self colorForWithoutBiosimilar text: 'Sin biosimilar aprobado'. + legendBox addColor: Color white text: 'Sin informacion publicada'. + legendBox build. + ^ legendBox view +] + +{ #category : #'data visualization' } +PublishedMedInfo >> showPUDataColorConventions [ + "I create a legend showing the color palette and properties used in the matrix sunburst for prescription drug data. + By convention this data starst where admin data ends and two first rows are 'special' admin data: name and variant of + medicament which are already mapped on" + + | legendBox | + + legendBox := RTLegendBuilder new. + legendBox addText: 'Convenciones' asUppercase. + + 1 to: (self medDataKeys size) - (self adminDataSize + 2) do: [ :i | + legendBox addColor: (self colorPalette26 at: (self colorPalette26 size + 1 - i)) text: ((self medDataKeys at: self adminDataSize + 2 + i) copyReplaceAll: '* ' with: ''). + ]. + legendBox addColor: self colorForWithoutBiosimilar text: 'Sin biosimilar aprobado'. + legendBox addColor: Color white text: 'Sin informacion publicada'. + legendBox build. + ^ legendBox view +] diff --git a/src/Dataviz/Tweet.class.st b/src/Dataviz/Tweet.class.st index 207acbe..28f5c63 100644 --- a/src/Dataviz/Tweet.class.st +++ b/src/Dataviz/Tweet.class.st @@ -11,7 +11,11 @@ Class { 'date', 'message', 'profile', - 'url' + 'url', + 'mentions', + 'links', + 'hashtags', + 'type' ], #category : #'Dataviz-Twitter' } @@ -26,6 +30,46 @@ Tweet >> date: anObject [ date := anObject ] +{ #category : #accessing } +Tweet >> hashtags [ + ^ hashtags +] + +{ #category : #accessing } +Tweet >> hashtags: anObject [ + hashtags := anObject +] + +{ #category : #'data scrapping' } +Tweet >> impact [ + "Computes the maximum amount of followers that a tweet can reach, including the retweets. + It doesn't take into account that retweeters can share followers and in future versions + this should be computed, even if we don't have a full list of followers. + Also the computing time could be sustancially improved by extracting only followers data + from the profiles instead of scrapping all and by having a better cache which includes + prescrapped data. + 'Real time' is not the main issue here, but 'slow data'. + For example we could include snapshots of the retweeters profiles and to aks for a + commented retweet to reach new followers" + + | retweetsHtml retweetsTree retweeters impactData | + + retweetsHtml := ZnClient new get: 'http://mutabit.com/deltas/repos.fossil/datapolis/doc/tip/Data/Sources/Single/', ((self url splitOn: '//') at: 2), '/retweets.html'. + + retweetsTree := Soup fromString: retweetsHtml. + + retweeters := (retweetsTree findAllTagsByClass: 'username') + collect: [:each | each text copyReplaceAll: '@' with: '']. + + impactData := Dictionary new. + + impactData + at: 'retweeters' put: retweeters size; + at: 'reach' put: (retweeters collect: [:each | (TwitterProfile new scrapDataForProfile: each) followers]) sum. + + ^ impactData. +] + { #category : #'data scrapping' } Tweet >> impactFor: aTweetUrl [ "Computes the maximum amount of followers that a tweet can reach, including the retweets. @@ -51,7 +95,7 @@ Tweet >> impactFor: aTweetUrl [ impactData at: 'retweeters' put: retweeters size; - at: 'reach' put: (retweeters collect: [:each | (TwitterProfile new scrapDataFromProfile: each) followers]) sum. + at: 'reach' put: (retweeters collect: [:each | (TwitterProfile new scrapFollowersForProfile: each)]) sum. ^ impactData. ] @@ -94,6 +138,62 @@ RTNest new on: background nest: els. ^ background ] +{ #category : #accessing } +Tweet >> links [ + ^ links +] + +{ #category : #accessing } +Tweet >> links: anObject [ + links := anObject +] + +{ #category : #accessing } +Tweet >> mentions [ + ^ mentions +] + +{ #category : #accessing } +Tweet >> mentions: anObject [ + mentions := anObject +] + +{ #category : #'data visualization' } +Tweet >> mentionsClusterSeparated: aDistance inView: aView [ + "I return a graphic of all avatars mentioned in a tweet, layered as a cluster a separated + by aDistance" + + | container avatars profiles | + + "Profiles: Create profiles collecting information for all users mentioned in the tweet" + profiles := OrderedCollection new. + self mentions do: [:eachMention | + profiles add: (TwitterProfile new scrapDataForProfile: eachMention)]. + + "Avatars: Use the profiles to create a collection of avatar images which are draggable + elements on a view" + avatars := OrderedCollection new. + profiles do: [:eachProfile | + avatars add: (RTBitmap new form: eachProfile avatar) element ]. + RTCircleLayout new + initialRadius: (avatars size)*aDistance; + on: avatars. + + container := (RTBox new color: Color transparent) element . + RTNest new on: container nest: avatars. + + "Adding avatars and its labels" + aView addAll: avatars. + aView add: container. + + 1 to: (avatars size) do: [:i | + (avatars at: i) @ (RTLabelled new + text: (('@',(self mentions at: i))); + fontSize: 35; + color: (Color black))]. + ^ container +] + { #category : #accessing } Tweet >> message [ ^ message @@ -114,33 +214,199 @@ Tweet >> profile: anObject [ profile := anObject ] +{ #category : #'data scrapping' } +Tweet >> scrapDataFromUrl: aTweetUrl [ + "Scraps most of the data in the page of a aTweetUrl. Most of the tweets are prestored now, but in the future + it will combine prestored data and live streaming data" + +| tweetSourceHtml tweetSourceTree hour dateTmp day month year| +tweetSourceHtml := ZnClient new get: + 'http://mutabit.com/deltas/repos.fossil/datapolis/doc/tip/Data/Sources/Single/', + ((aTweetUrl splitOn: '//') at: 2), '/tweet.html'. + +tweetSourceTree := Soup fromString: tweetSourceHtml. +self url: aTweetUrl . +self profile: ((aTweetUrl splitOn: '/') at: 4). +self message: ((((tweetSourceTree findAllTagsByClass: 'tweet') at: 1) children at: 6) children at: 2). +self mentions: ((self message findAllTagsByClass: 'twitter-atreply') + collect: [:each | (each attributeAt: 'href') copyReplaceAll: '/' with: '']). +self hashtags: ((self message findAllTagsByClass: 'twitter-hashtag') collect: [:each | each text]). +self links: ((self message findAllTagsByClass: 'twitter-timeline-link') + collect: [:each | each attributeAt: 'data-url']). +hour := ((((tweetSourceTree findAllTagsByClass: 'tweet') at: 1) children at: 8) children at: 2). +dateTmp := (((((tweetSourceTree findAllTagsByClass: 'tweet') at: 1) children at: 8) children at: 4) text) splitOn: ' '. +day := dateTmp at: 3. +month := dateTmp at: 4. +year := '20', (dateTmp at: 5). +self date: (year, month, day, hour text, hour nextSibling text) asDateAndTime. +] + { #category : #'data visualization' } -Tweet >> silenceMap [ - "Creates a visualization of how long a tweet has not been answered (any kind of answer: not favs, not RT, no nothing), as - 'circles of silence' " +Tweet >> showInView: aView [ + "shows the general information (author, author's avatar and text) of the tweet in a view" - | v circle ex1 ex2 line retweets reach diameter radious origin | + | tweetPanel tweetText holder avatar | + + "Tweet text" + (self message text size > 50) + ifTrue:[ + tweetText := RTLabel new + text: (self splitByLines at: 1), String cr, + (self splitByLines at: 2), String cr, String cr, + (self date asString)] + ifFalse: [tweetText := RTLabel new + text: (self splitByLines at: 1), String cr, String cr, + (self date asString) ]. + tweetText height: 35. + tweetText := tweetText element. + + "Avatar" + avatar := (RTBitmap new form: + ((TwitterProfile new scrapDataForProfile: self profile) avatar)) element. + avatar translateBy: (-30*(30)@0). + + "Putting tweet elements together" + tweetPanel := OrderedCollection new. + tweetPanel add: avatar; add: tweetText. + holder := (RTBox new color: Color transparent) element @ RTDraggable. + RTNest new on: holder nest: tweetPanel. + + "Adding elements and labels to the view" + aView add: tweetText. + aView add: avatar. + aView add: holder. + avatar @ (RTLabelled new + text: '@', (self profile); + fontSize: 35; + color: Color black; + top; + offsetOnEdge: 1 ). + ^ holder. +] -diameter := 60. -origin := (100@100). -radious := diameter / 2. -v := RTView new. +{ #category : #'data visualization' } +Tweet >> showInView: aView sized: aSize [ + "shows the general information (author, author's avatar and text) of the tweet in a view" -circle := (RTEllipse new size: diameter; borderColor: Color black; color: Color transparent) element. -ex1 := RTBox element. -ex2 := RTBox element. -circle translateTo: origin. -ex1 translateTo: ((radious negated @ 0) + circle center). -ex2 translateTo: (radious @ 0) + circle center. -line := (RTLine new; color: Color black) edgeFrom: ex1 to: ex2. -v add: circle; add: line. + | tweetPanel tweetText holder avatar | + + "Tweet text" + (self message text size > 50) + ifTrue:[ + tweetText := RTLabel new + text: (self splitByLines at: 1), String cr, + (self splitByLines at: 2), String cr, String cr, + (self date asString)] + ifFalse: [tweetText := RTLabel new + text: (self splitByLines at: 1), String cr, String cr, + (self date asString) ]. + tweetText height: aSize. + tweetText := tweetText element. + + "Avatar" + avatar := (RTBitmap new form: + ((TwitterProfile new scrapDataForProfile: self profile) avatar)) element. + avatar translateBy: (-30*aSize@0). + + "Putting tweet elements together" + tweetPanel := OrderedCollection new. + tweetPanel add: avatar; add: tweetText. + holder := (RTBox new color: Color transparent) element @ RTDraggable. + RTNest new on: holder nest: tweetPanel. + + "Adding elements and labels to the view" + aView add: tweetText. + aView add: avatar. + aView add: holder. + avatar @ (RTLabelled new + text: '@', (self profile); + fontSize: 35; + color: Color black; + top; + offsetOnEdge: 1 ). + ^ holder. +] -retweets := (RTLabel text: 5) element. -reach := (RTLabel text: '3.1k') element. -v add: retweets; add: reach. -retweets translateTo: (0 @ -12) + circle center. -reach translateTo: (0 @ 12) + circle center. -^ v. +{ #category : #'data visualization' } +Tweet >> silenceMapFor: arg1 [ + | tmp1 tmp2 tmp3 tmp4 tmp5 tmp6 tmp7 tmp8 tmp9 tmp11 tmp13 tmp15 tmp17 tmp19 tmp21 | + self scrapDataFromUrl: arg1. + tmp5 := (Date today - self date) days. + tmp7 := self impactFor: arg1. + tmp11 := RTBox new. + tmp11 color: Color red. + tmp3 := tmp11 size: 200. + tmp3 := tmp3 element. + tmp3 translateBy: (tmp5 * -50) @ 0. + tmp1 := RTView new. + tmp6 := self mentionsClusterSeparated: 70 inView: tmp1. + tmp2 := self showInView: tmp1 sized: 35. + tmp2 translateBy: (tmp5 * -25) @ -600. + tmp1 add: tmp3. + tmp4 := RTEdge from: tmp6 to: tmp3. + tmp13 := RTGradientColoredLine new. + tmp13 + colors: (Array with: (Color white alpha: 0.3) with: (Color red alpha: 0.9)); + precision: 100; + width: 20. + tmp1 add: tmp4 + tmp13 gradientColorShape. + tmp15 := RTLabel new. + tmp15 text: tmp5 asString , ' días sin respuesta'. + tmp8 := (tmp15 height: tmp5 * 2.5) element @ RTDraggable. + tmp8 translateBy: (tmp5 * -25) @ -105. + tmp17 := RTLabel new. + tmp17 + text: 'al ' , Date today asString , ' y contando...'; + height: tmp5 * 1. + tmp9 := (tmp17 color: Color gray) element @ RTDraggable. + tmp9 translateBy: (tmp5 * -15) @ 50. + tmp1 + add: tmp8; + add: tmp9. + tmp19 := RTLabelled new. + tmp19 + text: (tmp7 at: 'retweeters') asString , ' retweets'; + fontSize: tmp5 * 2. + tmp3 @ (tmp19 color: Color gray). + tmp21 := RTLabelled new. + tmp21 + text: (tmp7 at: 'reach') asString , ' lectores' , String cr , '(max)'; + fontSize: tmp5 * 2; + color: Color gray. + tmp3 @ tmp21 below. + tmp1 view canvas focusOnCenterScaled. + ^ tmp1 @ RTDraggableView +] + +{ #category : #'as yet unclassified' } +Tweet >> splitByLines [ + "Splits the message text of the tweet in lines, in case of being too long for a better rendering + in silenceMap and other views." + + | summa i charLimit words wordSizes lines | + + words := self message text splitOn: ' '. + wordSizes := words collect: [:eachWord | eachWord size]. + i := 1. + summa := 0. + charLimit := 50. + [summa < charLimit] whileTrue: [ + summa := (wordSizes at: i) + summa. + i := i +1]. + lines := OrderedCollection new. + lines add: (Character space join: (words copyFrom: 1 to: i)). + lines add: (Character space join: (words copyFrom: i + 1 to: words size)). + ^ lines +] + +{ #category : #accessing } +Tweet >> type [ + ^ type +] + +{ #category : #accessing } +Tweet >> type: anObject [ + type := anObject ] { #category : #accessing } diff --git a/src/Dataviz/TweetsCollection.class.st b/src/Dataviz/TweetsCollection.class.st new file mode 100644 index 0000000..07373e6 --- /dev/null +++ b/src/Dataviz/TweetsCollection.class.st @@ -0,0 +1,345 @@ +" +Please comment me using the following template inspired by Class Responsibility Collaborator (CRC) design: + +I'm Tweets a helper class to work with collections of tweet objects. + +For the Responsibility part: Three sentences about my main responsibility, what I'm doing, what services do I offer. + +For the Collaborators Part: State my main collaborators and one line about how I interact with them. + +Public API and Key Messages + +- message one +- message two +- what is the way to create instances is a plus. + + One simple example is simply gorgeous. + +Internal Representation and Key Implementation Points. + + Instance Variables + tweets: + + + Implementation Points +" +Class { + #name : #TweetsCollection, + #superclass : #Object, + #instVars : [ + 'tweets' + ], + #category : #'Dataviz-Twitter' +} + +{ #category : #'data visualization' } +TweetsCollection >> activityHistogramFor: aProfileName in: aDataBaseFile [ + "I draw a histogram of the tweeter activity for a given profile name with data stored in aDataBaseFile. + The database stores the individual tweets for this profile, with their type (tweet, retweet or reply), + unique url and date. + A proper schema of the data base still needs to be published. + Is the one used in all references to aDataBaseFile." + + | sample activityDataArray monthOfFirstTweet activityDataCollection histogramData plot | + sample := TweetsCollection new. + activityDataArray := sample monthlyActivityDataFor: aProfileName in: aDataBaseFile. + (activityDataArray size > 0) + ifFalse: [ + self inform: 'There is no data for ', aProfileName, ' in the database' + ] + ifTrue:[ + "Detecting where happened the first tweet and storing only retweets over this value, will delete the + outliers which correspond to retweets of original tweets far away of the period of the sample" + monthOfFirstTweet := activityDataArray detect: [ :each | (each value at: 1) > 0]. + activityDataCollection := OrderedCollection new. + activityDataArray do: [ :each | + (each key >= monthOfFirstTweet key) ifTrue: [ + activityDataCollection add: { + each key asString . + each value at: 1 . + each value at: 2 . + each value at: 3 } + ] + ]. + + "This part was adapted from the awesome roassal examples" + plot := RTGrapher new. + histogramData := RTMultipleData new. + histogramData barShape color: Color green. + histogramData + points: activityDataCollection; + addMetric: #second; + addMetric: #third; + addMetric: #fourth. + + "Horizontal text" + "d barChartWithBarCenteredTitle: #first." + + "Rotated text with integer axis" + histogramData barChartUsing: (RTBarLabelFactory new label: #first; fontSize: 7). + plot add: histogramData. + plot axisY noDecimal. + ^ plot + ] +] + +{ #category : #'data queries' } +TweetsCollection >> importTweetsFromJSONFile: aJSONFile [ + "I import all the tweets for aJSONFile and convert them in tweets inside a TweetCollection" + | stream truncated jsonData currentTweet | + stream := aJSONFile readStream. + "We need to truncate the original file to quite the first line, which is the name of the exported array, so NeoJSONReader doesn't complain" + truncated := WriteStream on: String new. + stream contents lines allButFirstDo: [ :each | truncated nextPutAll: each ]. + + jsonData := NeoJSONReader fromString: truncated contents asString. + jsonData do: [:each | + currentTweet := Tweet new. + currentTweet + message: (each at: 'text'); + profile: ((each at: 'user') at: 'screen_name'); + date: ((each at: 'created_at') copyFrom: 1 to: 19) asDateAndTime. + "Detecting the kind of message and processing accordingly" + (each keys includes: 'in_reply_to_status_id') + ifTrue: [ + currentTweet + type: 'reply'; + url: '/', (currentTweet profile, '/status/', (each at: 'id_str'))] + ifFalse: [(each keys includes: 'retweeted_status') + ifTrue: [ + currentTweet + type: 'retweet'; + url: '/', + (((each at: 'retweeted_status') at: 'user') at: 'screen_name'), + '/status/', + ((each at: 'retweeted_status') at: 'id_str'). + ] + ifFalse: [ + currentTweet + type: 'tweet'; + url: '/', (currentTweet profile, '/status/', (each at: 'id_str')) ] + ]. + "Detecting hashtags" + "(((each at: 'entities') at: 'hashtags') size > 0) + ifTrue: [ + (each at: 'entities') at: 'hashtags' + ]." + self tweets add: currentTweet. + ]. +] + +{ #category : #'data queries' } +TweetsCollection >> loadTweetsFor: aProfileName from: aDataBaseFile [ + "I select all the tweets for aProfileName in a given database" + | db queryResults temporalTweet | + + "openning connection" + db := NBSQLite3Connection on: aDataBaseFile. + db open. + "Querying the data base" + queryResults := (db execute: 'select * from tweets where profile="',aProfileName,'";') rows. + db close. + + queryResults do: [ :each | + temporalTweet := Tweet new. + temporalTweet + url: (each at: 'url'); + date: (TimeStamp fromUnixTime: (each at: 'date')) asUTC; + type: (each at: 'type'); + message: (each at: 'message'); + profile: (each at: 'profile'). + self tweets add: temporalTweet + ]. +] + +{ #category : #'data queries' } +TweetsCollection >> monthlyActivityDataFor: aProfileName in: aDataBaseFile [ + "I present a histogram of the tweets that differenciates tweets, retweets and replies, + for a given profile in a given SQLite database (for the moment I supposse that the profile exist there + and data base schema is correct)" + | db queryResults firstMonth lastMonth currentMonth activityCalendar monthOfFirstTweet | + + "openning connection" + db := NBSQLite3Connection on: aDataBaseFile. + db open. + "Querying the data base" + db execute: 'create temporary table profile_tweets as select * from tweets where profile="',aProfileName,'";'. + queryResults := (db execute: + 'SELECT strftime("%Y-%m",datetime(date, "unixepoch","localtime")) as month, type, count(*) as amount + FROM profile_tweets GROUP BY strftime("%Y-%m",datetime(date, "unixepoch","localtime")), type;') rows. + db execute: 'drop table if exists profile_tweets;'. + db close. + activityCalendar := Dictionary new. + (queryResults size > 0) + ifFalse: [ + self inform: 'There is no data for that profile in the database' + ] + ifTrue: [ + "Detecting where happened the first tweet and storing only retweets over this value, will delete the + outliers which correspond to retweets of original tweets far away of the period of the sample" + monthOfFirstTweet := queryResults detect: [ :each | (each at: 'month') notNil]. + firstMonth := ((monthOfFirstTweet at: 'month'), '-01') asDate asMonth. + lastMonth := ((queryResults last at: 'month'), '-01') asDate asMonth. + currentMonth := firstMonth. + [ currentMonth = (lastMonth next)] + whileFalse:[ + activityCalendar at: (currentMonth) put: { 0 . 0 . 0 }. + currentMonth := currentMonth next. + ]. + queryResults do: [ :each | + (each at: 'type') = 'tweet' & ((each at: 'month') notNil) ifTrue: [ + currentMonth := ((each at: 'month'), '-01') asDate asMonth. + (activityCalendar at: currentMonth) + at: 1 + put: (((activityCalendar at: currentMonth) at: 1) + (each at: 'amount')) + ]. + (each at: 'type') = 'retweet' ifTrue: [ + currentMonth := ((each at: 'month'), '-01') asDate asMonth. + (activityCalendar at: currentMonth) + at: 2 + put: (((activityCalendar at: currentMonth) at: 2) + (each at: 'amount')) + ]. + (each at: 'type') = 'reply' ifTrue: [ + currentMonth := ((each at: 'month'), '-01') asDate asMonth. + (activityCalendar at: currentMonth) + at: 3 + put: (((activityCalendar at: currentMonth) at: 3) + (each at: 'amount')) + ] + ]. + ]. + ^ (activityCalendar associations sorted) +] + +{ #category : #'data storage / persistance' } +TweetsCollection >> populateDataBase: aDataBaseFile [ + "I populate a SQLite database file with my tweets data" + | db | + "openning connection" + db := NBSQLite3Connection on: aDataBaseFile. + db open. + "Creating the data base tweets schema" + db execute: + 'create table if not exists tweets ( + url text primary key, + profile text, + date integer, + type text, + message text + );'. + "Populating the database" + self tweets do: [:each | + db execute: 'INSERT INTO tweets values (?, ?, ?, ?, ?);' + with: { + each url . + each profile . + each date ifNotNil: [each date asUnixTime asString ]. + each type . + each message }. + ]. + db close. +] + +{ #category : #'data visualization' } +TweetsCollection >> ringOverview [ + "I present a overview of the tweets as a ring that differenciates tweets, retweets and replies" + | totalTweets replies retweets ring | + replies := 0. + retweets := 0. + tweets do: [ :each | + (each type = 'reply') ifTrue: [replies := replies + 1]. + (each type = 'retweets') ifTrue: [retweets := retweets + 1]]. + totalTweets := (self tweets size) - replies - retweets. + ring := RTPieBuilder new. + ring interaction popup. + ring shape current + innerRadius: 80; + externalRadius: 100. + ring objects: {totalTweets . retweets . replies}. + (ring slice: #value)ifNotNil: [ :group | + group do: [:each | each @ (RTDraggable groupToDrag: group)] + ]. + ring normalizer distinctColor. + ring build. + ^ ring. +] + +{ #category : #'data visualization' } +TweetsCollection >> ringOverviewFor: aProfileName in: aDataBaseFile [ + "I present a overview of the tweets as a ring that differenciates tweets, retweets and replies, + for a given profile in a given SQLite database (for the moment I supposse that the profile exist there + and data base schema is correct)" + | db totalTweets replies retweets ring | + + "openning connection" + db := NBSQLite3Connection on: aDataBaseFile. + db open. + "Querying the data base" + retweets := (db execute: 'select * from tweets where profile="', aProfileName ,'" and type="retweet";') rows size. + replies := (db execute: 'select * from tweets where profile="', aProfileName ,'" and type="reply";') rows size. + totalTweets := (db execute: 'select * from tweets where profile="', aProfileName ,'";') rows size - retweets - replies. + db close. + (totalTweets > 0) + ifFalse: [ + self inform: 'There are no tweets in the database for that profile' + ] + ifTrue: [ + ring := RTPieBuilder new. + ring interaction popup. + ring shape current + innerRadius: 80; + externalRadius: 100. + ring objects: {totalTweets . retweets . replies}. + (ring slice: #value)ifNotNil: [ :group | + group do: [:each | each @ (RTDraggable groupToDrag: group)] + ]. + ring normalizer distinctColor. + ring build. + ^ ring]. +] + +{ #category : #'data scrapping' } +TweetsCollection >> scrapTweetsFromHtmlFile: aHtmlFile [ + "I scraps tweets from a downloaded html file. + On how to download such file for any given public twitter profile look at: + http://blog.databigbang.com/scraping-web-sites-which-dynamically-load-data/ + " + | tweetsDump htmlTree tweetsHtml tweet unixTime answersArray tweetsTemp profile | + + tweetsDump := aHtmlFile readStream. + htmlTree := Soup fromString: tweetsDump contents. + profile := (((htmlTree findAllTagsByClass: 'ProfileHeaderCard-screennameLink') at: 1) attributeAt: 'href') copyReplaceAll: '/' with: ''. + tweetsHtml := htmlTree findAllTagsByClass: 'tweet'. + tweetsTemp := OrderedCollection new. + tweetsHtml allButLastDo: [:each | + tweet := Tweet new. + (each findAllTagsByClass: '_timestamp') size > 0 + ifTrue: [ + unixTime := (((each findAllTagsByClass: '_timestamp') at: 1) attributeAt: 'data-time') asInteger. + tweet date: (TimeStamp fromUnixTime: unixTime) asUTC + ]. + answersArray := each findAllTagsByClass: 'js-retweet-text'. + (answersArray size = 1) + ifTrue: [tweet type: 'retweet'] + ifFalse: [ + (each attributeAt: 'data-is-reply-to') isString + ifTrue: [tweet type: 'reply'] + ifFalse: [tweet type: 'tweet'] + ]. + tweet url: (each attributeAt: 'data-permalink-path'). + (each findAllTagsByClass: 'TweetTextSize') size > 0 + ifTrue: [tweet message: (((each findAllTagsByClass: 'TweetTextSize') at: 1) text)]. + tweet profile: profile. + tweetsTemp add: tweet. + ]. + self tweets: tweetsTemp. +] + +{ #category : #accessing } +TweetsCollection >> tweets [ + ^ tweets ifNil: [tweets := OrderedCollection new] +] + +{ #category : #accessing } +TweetsCollection >> tweets: anOrderedCollection [ + tweets := anOrderedCollection +] diff --git a/src/Dataviz/TwitterProfile.class.st b/src/Dataviz/TwitterProfile.class.st index d201136..e2bd59e 100644 --- a/src/Dataviz/TwitterProfile.class.st +++ b/src/Dataviz/TwitterProfile.class.st @@ -162,6 +162,26 @@ TwitterProfile >> lastTweets: anObject [ lastTweets := anObject ] +{ #category : #'data storage / persistance' } +TwitterProfile >> loadDataFor: aProfileName fromDatabase: aDataBaseFile [ + | db queryResults | + + "openning connection" + db := NBSQLite3Connection on: aDataBaseFile. + db open. + "Querying the data base" + queryResults := (db execute: 'SELECT * FROM profiles WHERE screenName="',aProfileName,'";') rows at: 1. + db close. + self + screenName: (queryResults at: 'screenName'); + name: (queryResults at: 'name'); + avatar: (queryResults at: 'avatar'); + bio: (queryResults at: 'bio'); + favs: (queryResults at: 'favs'); + followers: (queryResults at: 'followers'); + following: (queryResults at: 'following') +] + { #category : #'data storage / persistance' } TwitterProfile >> loadDataFromFile: aFileReference [ "Opens the twitter profile from aFileReference stored in the STON format" @@ -226,6 +246,39 @@ TwitterProfile >> name: anObject [ name := anObject ] +{ #category : #'data storage / persistance' } +TwitterProfile >> populateDataBase: aDataBaseFile [ + "I populate a SQLite database file with myself data" + | db | + "openning connection" + db := NBSQLite3Connection on: aDataBaseFile. + db open. + "Creating the data base tweets schema" + db execute: + 'create table if not exists profiles ( + screenName text primary key, + name text, + avatar blob, + bio text, + favs integer, + followers integer, + following integer, + location text + );'. + "Populating the database" + db execute: 'INSERT INTO profiles values (?, ?, ?, ?, ?, ?, ?, ?);' + with: { + self screenName. + self name. + self avatar. + self bio. + self favs. + self followers. + self following. + self location}. + db close. +] + { #category : #'data storage / persistance' } TwitterProfile >> saveToFile: aFileReference [ "Saves the twitter profile to aFileReference in the STON format" @@ -235,6 +288,24 @@ TwitterProfile >> saveToFile: aFileReference [ stream nextPutAll: (STON toStringPretty: self). ] +{ #category : #dataweek } +TwitterProfile >> sayBye [ + "Just says hello to all the people which is listening. A dummy example on how to create new messages" + Transcript open. + Transcript show: 'Adios, perfil de Twitter, despidiéndose. Pásala bueno ;-)' + + +] + +{ #category : #dataweek } +TwitterProfile >> sayHello [ + "Just says hello to all the people which is listening. A dummy example on how to create new messages" + Transcript open. + Transcript show: 'Hola! soy un perfil de Twitter :-)' + + +] + { #category : #'data scrapping' } TwitterProfile >> scrapAvatarFrom: aHtmlString [ "Finds the avatar in a twitter's main page profile, scales it (200x200), cast it agains different formats (jpeg, png) and returns it" @@ -245,15 +316,14 @@ TwitterProfile >> scrapAvatarFrom: aHtmlString [ (avatarUrl asLowercase endsWith: '.png') ifTrue: [avatarImage := ZnEasy getPng: avatarUrl]. ((avatarUrl asLowercase endsWith: '.jpeg') or: (avatarUrl asLowercase endsWith: '.jpg')) - ifTrue: [ - avatarImage := ZnEasy get: avatarUrl. - "(PNGReadWriter on: avatarImage ) nextPutImage: (JPEGReadWriter on: avatarImage )" ]. + ifTrue: [avatarImage := ZnEasy getJpeg: avatarUrl]. ^avatarImage. ] { #category : #'data scrapping' } -TwitterProfile >> scrapDataFromProfile: aProfileName [ - "Scraps data from aProfileName and fills out the TwitterProfile. The profile name is the last part of a twitter profile url +TwitterProfile >> scrapDataForProfile: aProfileName [ + "Scraps data from aProfileName and fills out the TwitterProfile. + The profile name is the last part of a twitter profile url (i.e: 'https://twitter.com/aProfileName')." | client source numericalData anUrl | @@ -282,6 +352,24 @@ TwitterProfile >> scrapDataFromProfile: aProfileName [ ] +{ #category : #'data scrapping' } +TwitterProfile >> scrapFollowersForProfile: aProfileName [ + "Scraps data from a predefined profile name" + + | client source numericalData anUrl | + anUrl := 'https://twitter.com/', aProfileName. + client := ZnClient new. + client get: anUrl. + client isSuccess + ifTrue:[ + source := Soup fromString: (client) contents asString. + numericalData := (source findAllTagsByClass: 'ProfileNav-value') collect:[:each | each text]. + followers := self asNumber: (numericalData at: 3). + ] + ifFalse:[self inform: 'Algo salió mal. Verifique su conexión a Internet y que el contenido buscado estén disponibles']. + ^ self followers +] + { #category : #'data scrapping' } TwitterProfile >> scrapTweetsFromFile: aFile [ "Scraps tweets data from aFile, wich contains tweets scrapped from a public profile."