Private Instagram User Scraper for Diggernaut

---
config:
    agent: Firefox
iterator:
    - type: fieldset
      fields:
      - user: somusername
        itemstoget: 30
        mode: simple
do:
- walk:
    to: https://www.instagram.com/
    do:
    - find:
        path: body
        do:
        - parse:
            filter: window\._sharedData\s+\=\s+([^;]+);
        - normalize:
            routine: json2xml
        - to_block
        - find:
            path: config>csrf_token
            do:
            - parse
            - variable_set: token
        - walk:
            to:
                post: https://www.instagram.com/accounts/login/ajax/
                headers:
                    x-csrftoken: <%token%>
                    x-instagram-ajax: 1
                    x-requested-with: XMLHttpRequest
                data:
                    username: your_instagram_username
                    password: your_instagram_password
            do:
            - walk:
                to: https://www.instagram.com/<%user%>/
                do:
                - find:
                    path: body
                    do:
                    - parse:
                        filter: window\._sharedData\s+\=\s+([^;]+);
                    - normalize:
                        routine: json2xml
                    - to_block
                    - find:
                        path: config>csrf_token
                        do:
                        - parse
                        - variable_set: token
                    - find:
                        path: entry_data>profilepage
                        do:
                        - register_set: https://www.instagram.com/p
                        - variable_set: baseurl
                        - object_new: user
                        - find:
                            path: user>id
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: id
                            - variable_set: userid
                        - find:
                            path: user>username
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: username
                        - find:
                            path: user>full_name
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: full_name
                        - find:
                            path: user>biography
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: biography
                        - find:
                            path: user>profile_pic_url
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: profile_pic_url
                        - find:
                            path: user>profile_pic_url_hd
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: profile_pic_url_hd
                        - find:
                            path: user>external_url
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: external_url
                        - find:
                            path: user>external_url_linkshimmed
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: external_url_linkshimmed
                        - find:
                            path: user>country_block
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: country_block
                        - find:
                            path: user>follows>count
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: follows
                            - variable_set: follows
                        - find:
                            path: user>followed_by>count
                            do:
                            - parse
                            - object_field_set:
                                object: user
                                field: followed_by
                            - variable_set: followed_by
                        - find:
                            path: user>media>page_info>start_cursor
                            do:
                            - parse
                            - variable_set: cursor
                        - argument_get: itemstoget
                        - if:
                            match: all
                            do:
                            - find:
                                path: user>media>count
                                do:
                                - parse
                                - variable_set: items_get
                            else:
                            - variable_set: items_get
                        - walk:
                            to:
                                post: https://www.instagram.com/query/
                                headers:
                                    x-csrftoken: <%token%>
                                    x-instagram-ajax: 1
                                    x-requested-with: XMLHttpRequest
                                data:
                                    q: 'ig_user(<%userid%>) {
  followed_by.first(<%followed_by%>) {
    count,
    page_info {
      end_cursor,
      has_next_page
    },
    nodes {
      id,
      is_verified,
      followed_by_viewer,
      requested_by_viewer,
      full_name,
      profile_pic_url,
      username
    }
  }
}'
                                    ref: 'relationships::follow_list'
                            do:
                            - find:
                                path: followed_by>nodes
                                do:
                                - object_new: followers
                                - find:
                                    path: id
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: followers
                                        field: id
                                - find:
                                    path: username
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: followers
                                        field: username
                                - find:
                                    path: full_name
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: followers
                                        field: full_name
                                - find:
                                    path: profile_pic_url
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: followers
                                        field: profile_pic_url
                                - find:
                                    path: is_verified
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: followers
                                        field: is_verified
                                - find:
                                    path: followed_by_viewer
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: followers
                                        field: followed_by_viewer
                                - find:
                                    path: requested_by_viewer
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: followers
                                        field: requested_by_viewer
                                - object_save:
                                    name: followers
                                    to: user
                        - walk:
                            to:
                                post: https://www.instagram.com/query/
                                headers:
                                    x-csrftoken: <%token%>
                                    x-instagram-ajax: 1
                                    x-requested-with: XMLHttpRequest
                                data:
                                    q: 'ig_user(<%userid%>) {
  follows.first(<%follows%>) {
    count,
    page_info {
      end_cursor,
      has_next_page
    },
    nodes {
      id,
      is_verified,
      followed_by_viewer,
      requested_by_viewer,
      full_name,
      profile_pic_url,
      username
    }
  }
}'
                                    ref: 'relationships::follow_list'
                            do:
                            - find:
                                path: follows>nodes
                                do:
                                - object_new: following
                                - find:
                                    path: id
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: following
                                        field: id
                                - find:
                                    path: username
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: following
                                        field: username
                                - find:
                                    path: full_name
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: following
                                        field: full_name
                                - find:
                                    path: profile_pic_url
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: following
                                        field: profile_pic_url
                                - find:
                                    path: is_verified
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: following
                                        field: is_verified
                                - find:
                                    path: followed_by_viewer
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: following
                                        field: followed_by_viewer
                                - find:
                                    path: requested_by_viewer
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: following
                                        field: requested_by_viewer
                                - object_save:
                                    name: following
                                    to: user
                        - walk:
                            to:
                                post: https://www.instagram.com/query/
                                headers:
                                    x-csrftoken: <%token%>
                                    x-instagram-ajax: 1
                                    x-requested-with: XMLHttpRequest
                                data:
                                    q: 'ig_user(<%userid%>) { media.after(<%cursor%>, <%items_get%>) {
  count,
  nodes {
    caption,
    code,
    comments {
      count
    },
    comments_disabled,
    date,
    dimensions {
      height,
      width
    },
    display_src,
    id,
    is_video,
    likes {
      count
    },
    owner {
      id
    },
    thumbnail_src,
    video_views
  },
  page_info
}
 }'
                                    ref: 'users::show'
                            do:
                            - find:
                                path: media>count
                                do:
                                - parse
                                - object_field_set:
                                    object: user
                                    field: media_count
                            - find:
                                path: media>nodes
                                do:
                                - object_new: nodes
                                - find:
                                    path: id
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: id
                                - find:
                                    path: is_video
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: is_video
                                - find:
                                    path: video_views
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: vide_views
                                - find:
                                    path: date
                                    do:
                                    - parse
                                    - normalize:
                                        routine: date_format
                                        args:
                                            format_in: '%s'
                                            format_out: '%Y-%m-%d %H:%M:%S'
                                    - object_field_set:
                                        object: nodes
                                        field: date
                                - find:
                                    path: dimensions>width
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: width
                                - find:
                                    path: dimensions>height
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: height
                                - find:
                                    path: likes>count
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: likes_count
                                - find:
                                    path: comments>count
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: comments_count
                                - find:
                                    path: comments_disabled
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: comments_disabled
                                - find:
                                    path: caption_safe
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: caption
                                - find:
                                    path: thumbnail_src
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: thumbnail
                                - find:
                                    path: display_src
                                    do:
                                    - parse
                                    - object_field_set:
                                        object: nodes
                                        field: media
                                - find:
                                    path: code
                                    do:
                                    - parse
                                    - variable_prepend:
                                        field: baseurl
                                        joinby: "/"
                                    - object_field_set:
                                        object: nodes
                                        field: url
                                    - variable_set: node_url
                                    - argument_get: mode
                                    - if:
                                        match: extended
                                        do:
                                        - variable_get: node_url
                                        - walk:
                                            to: value
                                            do:
                                            - find:
                                                path: body
                                                do:
                                                - parse:
                                                    filter: window\._sharedData\s+\=\s+([^;]+);
                                                - normalize:
                                                    routine: json2xml
                                                - to_block
                                                - find:
                                                    path: entry_data>postpage>media>comments>nodes
                                                    do:
                                                    - object_new: comments
                                                    - find:
                                                        path: '>id'
                                                        do:
                                                        - parse
                                                        - object_field_set:
                                                            object: comments
                                                            field: id
                                                    - find:
                                                        path: user>id
                                                        do:
                                                        - parse
                                                        - object_field_set:
                                                            object: comments
                                                            field: user_id
                                                    - find:
                                                        path: user>username
                                                        do:
                                                        - parse
                                                        - object_field_set:
                                                            object: comments
                                                            field: username
                                                    - find:
                                                        path: user>profile_pic_url
                                                        do:
                                                        - parse
                                                        - object_field_set:
                                                            object: comments
                                                            field: user_profile_pic
                                                    - find:
                                                        path: text
                                                        do:
                                                        - parse
                                                        - object_field_set:
                                                            object: comments
                                                            field: text
                                                    - find:
                                                        path: created_at
                                                        do:
                                                        - parse:
                                                            filter: (\d+)\.\d+
                                                        - normalize:
                                                            routine: date_format
                                                            args:
                                                                format_in: '%s'
                                                                format_out: '%Y-%m-%d %H:%M:%S'
                                                        - object_field_set:
                                                            object: comments
                                                            field: date
                                                    - object_save:
                                                        name: comments
                                                        to: nodes
                                                - find:
                                                    path: entry_data>postpage>media>likes>nodes
                                                    do:
                                                    - object_new: likes
                                                    - find:
                                                        path: user>id
                                                        do:
                                                        - parse
                                                        - object_field_set:
                                                            object: likes
                                                            field: user_id
                                                    - find:
                                                        path: user>username
                                                        do:
                                                        - parse
                                                        - object_field_set:
                                                            object: likes
                                                            field: username
                                                    - find:
                                                        path: user>profile_pic_url
                                                        do:
                                                        - parse
                                                        - object_field_set:
                                                            object: likes
                                                            field: user_profile_pic
                                                    - object_save:
                                                        name: likes
                                                        to: nodes
                                                - sleep: 3
                                - object_save:
                                    name: nodes
                                    to: user
                        - object_save:
                                name: user