Advertisement
sxiii

Golos.io Articles To CSV Converter

Feb 16th, 2017
267
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 2.27 KB | None | 0 0
  1. #!/bin/bash
  2. # Golos.io Articles To CSV Converter
  3. # This script works in 3 parts:
  4. # 1. Loads user's articles from GOLOS.io.
  5. # 2. Loads each articles statistics.
  6. # 3. Converts statistics into one CSV file
  7. # Save as like "articles.sh", do "chmod +x articles.sh", and then run script like this:
  8. # ./articles.sh sxiii
  9. # Replace sxiii with your GOLOS.io nickname
  10. # Author: @sxiii
  11. # Date: 17.02.2017; 05:00 AM
  12.  
  13. tempfile="temp.articles.temp"    # Temp file (for articles list)
  14. nu="http://node.golos.ws"        # Node URL
  15. af="articles"                    # Article folder
  16. mkdir $af
  17.  
  18. echo "Welcome! Let's look at articles list of user @$1 - loading articles."
  19.  
  20. # Loading articles list from user $1
  21. x=$(curl -s --data '{"method": "get_discussions_by_author_before_date", "params": ["'$1'","","2016-06-06T00:00:00","100"], "id": 1 }' $nu)
  22. echo $x | jq -r 'recurse(.result.url[])?' | grep "url\":" | awk -F\" '{ print $4 }' | awk -F"/" '{ print $4 }' >> $tempfile
  23.  
  24. echo "Finished, I found total of "$(cat $tempfile | wc -l)" articles of user "$1
  25.  
  26. # Creating a template for CSV file
  27. t=0
  28. echo -e "\"Title\",\"Date\",\"Comments\",\"Votes\",\"Gold\",\"Tag1\",\"Tag2\",\"Tag3\",\"Tag4\",\"Tag5\",\"Images\",\"Text length\"" >> $af/allarticles.$1
  29.  
  30. # Starting a loop over $tempfile
  31. while read p; do
  32. ((t++))
  33. echo "Saving article $t: $p"
  34. y=$(curl -s --data '{"method": "get_content", "params": ["'$1'","'$p'"], "id": 1 }' $nu)
  35. echo $y | jq -r '.result.title,.result.created,.result.children,.result.net_votes,.result.total_payout_value,[.result.json_metadata|fromjson.tags],[.result.json_metadata|fromjson.image|length],[.result.body|length]' > $af/$p.$1
  36.  
  37. # Following code takes data from saved articles file and builds an index article file
  38. title=$(sed '1q;d' $af/$p.$1)
  39. time=$(sed '2q;d' $af/$p.$1)
  40. comments=$(sed '3q;d' $af/$p.$1)
  41. votes=$(sed '4q;d' $af/$p.$1)
  42. gold=$(sed '5q;d' $af/$p.$1)
  43. t1=$(sed '8q;d' $af/$p.$1)
  44. t2=$(sed '9q;d' $af/$p.$1)
  45. t3=$(sed '10q;d' $af/$p.$1)
  46. t4=$(sed '11q;d' $af/$p.$1)
  47. t5=$(sed '12q;d' $af/$p.$1)
  48. images=$(sed '16q;d' $af/$p.$1)
  49. symbols=$(sed '19q;d' $af/$p.$1)
  50.  
  51. echo -e "\"$title\",\"$time\",\"$comments\",\"$votes\",\"$gold\",$t1$t2$t3$t4$t5,\"$images\",\"$symbols\"" >> $af/allarticles.$1
  52.  
  53. done < $tempfile
  54.  
  55. # Remove tempfile if you like:
  56. rm $tempfile
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement