Scraping burton.com with Diggernaut

---
config:
    debug: 2
    agent: Firefox

do:
- link_add:
    pool: main
    url:
        - https://www.burton.com
- walk:
    to: links
    pool: main
    do:
    - variable_clear: ur
    - find:
        path: .category-title.secondary, .category-title.tertiary
        do:
        - parse:
            attr: href
        - variable_set: ur
        - register_set: <%ur%>?sz=999&concise=true&format=ajax
        - pool_clear: cat
        - link_add:
            pool: cat
        - walk:
            to: value
            headers:
                x-requested-with: XMLHttpRequest
            do:
            - find:
                path: body_safe hit > href
                do:
                - parse
                - normalize:
                    routine: url
                - link_add:
                    pool: sub

- walk:
    to: links
    pool: sub
    do:
    - find:
        path: .cps-app-page-body
        do:
        - variable_clear: allli
        - variable_clear: sdecr
        - variable_clear: li
        - variable_clear: ean
        - variable_clear: vid
        - variable_clear: id
        - variable_clear: stp
        - object_new: product
        - eval:
            routine: js
            body: '(function (){var d = new Date(); return d.toISOString()})();'
        - object_field_set:
            object: product
            field: date
        - static_get: url
        - object_field_set:
            object: product
            field: url
        - register_set: Burton
        - object_field_set:
            object: product
            field: brand
        - find:
            in: doc
            path: script:matches(window\.__bootstrap)
            do:
            - parse
            - trim
            - space_dedupe
            - eval:
                routine: js
                body: (function () {var window = {}; <%register%> return JSON.stringify(window.__bootstrap.data)})();
            - normalize:
                routine: json2xml
            - to_block
            - find:
                path: body_safe
                do:
                - node_remove: recommendations
                - find:
                    path: breadcrumbs
                    do:
                    - parse
                    - normalize:
                        routine: url
                    - walk:
                        to: value
                        do:
                        - find:
                            path: displayname
                            slice: 0:-2
                            do:
                            - parse
                            - space_dedupe
                            - trim
                            - object_field_set:
                                object: product
                                field: category
                                joinby: "|"
                - find:
                    path: variationcolor > displayvalue
                    do:
                    - parse
                    - trim
                    - space_dedupe
                    - if:
                        match: (\S)
                        do:
                        - object_field_set:
                            object: product
                            field: variations
                            joinby: "|"
                - find:
                    path: masterid
                    do:
                    - parse
                    - trim
                    - space_dedupe
                    - object_field_set:
                        object: product
                        field: sku
                    - variable_set: id
                    - walk:
                        to: https://www.burton.com/on/demandware.store/Sites-Burton_NA-Site/en_US/Product-GetTechFeaturesJSON?pids=<%id%>
                        do:
                        - find:
                            path: longdescription
                            do:
                            - parse
                            - space_dedupe
                            - trim
                            - if:
                                match: \S
                                do:
                                - object_field_set:
                                    object: product
                                    field: description
                    - walk:
                        to: https://www.burton.com/on/demandware.store/Sites-Burton_NA-Site/en_US/Product-GetVariationJSON?pid=<%id%>&pricing
                        do:
                        - find:
                            path: variationcolor displayname
                            do:
                            - parse
                            - space_dedupe
                            - trim
                            - if:
                                match: (\S)
                                do:
                                - object_field_set:
                                    object: product
                                    field: variations
                                    joinby: "|"
                        - find:
                            path: pricing
                            do:
                            - variable_clear: isSale
                            - find:
                                path: isonsale
                                do:
                                - parse
                                - if:
                                    match: 'true'
                                    do:
                                    - variable_set: isSale
                            - find:
                                path: saleprice
                                do:
                                - variable_get: isSale
                                - if:
                                    match: \S
                                    do:
                                    - parse:
                                        filter:
                                            - ^\s*\$(\d+\.?\d*)\s*-
                                            - ^\s*\$(\d+\.?\d*)
                                    - if:
                                        match: \d
                                        do:

                                        - object_field_set:
                                            object: product
                                            field: price
                                            type: float
                                        - register_set: USD
                                        - object_field_set:
                                            object: product
                                            field: currency
                            - find:
                                path: standardprice
                                do:
                                - variable_get: isSale
                                - if:
                                    match: \S
                                    else:
                                    - parse:
                                        filter:
                                            - ^\s*\$(\d+\.?\d*)\s*-
                                            - ^\s*\$(\d+\.?\d*)
                                    - if:
                                        match: \d
                                        do:

                                        - object_field_set:
                                            object: product
                                            field: price
                                            type: float
                                        - register_set: USD
                                        - object_field_set:
                                            object: product
                                            field: currency
                - find:
                    path: variationimagedata xl > x1, imagedata xl > x1, images xl > x1
                    do:
                    - parse
                    - trim
                    - space_dedupe
                    - filter:
                        args:
                            - (.+\.png)\?
                    - object_field_set:
                        object: product
                        field: images
                        joinby: "|"

                - find:
                    path:  name
                    slice: 0
                    do:
                    - parse
                    - trim
                    - space_dedupe
                    - object_field_set:
                        object: product
                        field: name
        - object_save:
            name: product