Scraping adidashardware.com with Diggernaut

---
config:
    debug: 2
    agent: Firefox
do:
- link_add:
    pool: main
    url:
        - https://www.adidashardware.com/training
        - https://www.adidashardware.com/recovery
        - https://www.adidashardware.com/strength
        - https://www.adidashardware.com/yoga
        - https://www.adidashardware.com/benches
        - https://www.adidashardware.com/cardio
- walk:
    to: links
    pool: main
    do:
    - find:
        path: script:matches(Wix Stores)
        do:
        - parse
        - space_dedupe
        - trim
        - filter:
            args:
                - var\s*rendererModel\s*=\s*(.+)\s*;\s*var\s*publicModel
        - normalize:
            routine: json2xml
        - to_block
        - find:
            path: clientspecmap > *:hasChild(appdefinitionname:matches(Wix Stores))  instance
            do:
            - parse
            - variable_set: instance
    - find:
        path: body
        do:
        - static_get: url
        - normalize:
            routine: replace_substring
            args:
                \s*$: ?_escaped_fragment_=
        - walk:
            to: value
            do:
            - find:
                path: meta[property="og:title"]
                do:
                - parse:
                    attr: content
                    filter:
                        - \s*\|\s*(.+)
                - normalize:
                    routine: lower
                - variable_set: cats
            - find:
                path: li[itemprop="itemListElement"] h3[itemprop="name"] > a
                do:
                - variable_clear: uri
                - parse:
                    attr: href
                - normalize:
                    routine: url
                - variable_set: uri
                - variable_clear: ur
                - filter:
                    args:
                        - product-page\/(.+)
                - variable_set: ur
                - walk:
                    to: https://ecom.wix.com/storefront/product/<%ur%>?instance=<%instance%>&locale=en
                    do:
                    - object_new: product
                    - find:
                        path: head
                        do:
                        - variable_get: cats
                        - object_field_set:
                            object: product
                            field: categories
                        - register_set: Adidas
                        - object_field_set:
                            object: product
                            field: brand
                        - eval:
                            routine: js
                            body: '(function (){var d = new Date(); return d.toISOString()})();'
                        - object_field_set:
                            object: product
                            field: date
                        - variable_get: uri
                        - object_field_set:
                            object: product
                            field: url
                    - find:
                        path: script:matches(eCom.eComAppConfig)
                        do:
                        - parse
                        - space_dedupe
                        - trim
                        - filter:
                            args:
                                - eCom\.eComAppConfig\(\'productPageApp\'\,\s*(.+),\s*\'\/\/
                        - normalize:
                            routine: json2xml
                        - to_block
                        - find:
                            path: body_safe
                            do:
                            - find:
                                path: product > description
                                do:
                                - parse
                                - space_dedupe
                                - trim
                                - normalize:
                                    routine: replace_substring
                                    args:
                                        \<p\>\s*\<\/p\>: ''
                                - register_set: <div><%register%></div>
                                - to_block
                                - find:
                                    path:  div
                                    do:
                                    - attr_remove:
                                        selector: '*'
                                    - parse
                                    - object_field_set:
                                        object: product
                                        field: description
                            - find:
                                path: product > media:hasChild(mediatype:matches(PHOTO)) > url
                                do:
                                - parse
                                - register_set: https://static.wixstatic.com/media/<%register%>
                                - object_field_set:
                                    object: product
                                    field: images
                                    joinby: "|"
                            - find:
                                path: product > name
                                do:
                                - parse
                                - space_dedupe
                                - trim
                                - object_field_set:
                                    object: product
                                    field: name
                            - find:
                                path: product options:hasChild(optiontype:matches(COLOR)) description
                                do:
                                - parse
                                - space_dedupe
                                - trim
                                - object_field_set:
                                    object: product
                                    field: variations
                                    joinby: "|"

                            - find:
                                path: product > sku, product > managedproductitems > sku
                                do:
                                - parse
                                - space_dedupe
                                - trim
                                - if:
                                    match: (\S{3})
                                    do:
                                    - object_field_set:
                                        object: product
                                        field: sku
                            - find:
                                path: product > prices > price
                                do:
                                - parse:
                                    filter:
                                        - (\d+\.?\d*)
                                - object_field_set:
                                    object: product
                                    field: price
                                    type: float
                                - register_set: GBP
                                - object_field_set:
                                    object: product
                                    field: currency
                    - object_save:
                        name: product