Scraping americanapparel.net with Diggernaut

---
config:
    debug: 2
    agent: Firefox
do:
- link_add:
    url: http://store.americanapparel.net
- link_add:
    url: http://store.americanapparel.net/en/factory-store/
- walk:
    to: links
    do:
    - find:
        path: .cd-primary-nav a
        do:
        - parse:
            attr: href
        - normalize:
            routine: url
        - link_add:
            pool: main
- walk:
    to: links
    pool: main
    do:
    - find:
        path: .product > a
        do:
        - parse:
            attr: href
        - normalize:
            routine: url
        - link_add:
            pool: sub
- walk:
    to: links
    pool: sub
    do:
    - sleep: 3
    - find:
        path: .pdp
        do:
        - variable_clear: allli
        - variable_clear: descr
        - variable_clear: li
        - variable_clear: id
        - variable_clear: views
        - variable_clear: color
        - variable_clear: imgnum
        - variable_clear: imgxl
        - variable_clear: viewsnum
        - variable_clear: stp
        - object_new: product
        - eval:
            routine: js
            body: '(function (){var d = new Date(); return d.toISOString()})();'
        - object_field_set:
            object: product
            field: date
        - static_get: url
        - object_field_set:
            object: product
            field: url
        - find:
            in: doc
            path: head meta[name="description"]
            do:
            - parse:
                attr: content
            - space_dedupe
            - trim
            - to_block
            - node_replace:
                path: br
                with: "\n"
            - split:
                context: text
                delimiter: \n+
            - find:
                path: div.splitted
                slice: 0
                do:
                - parse
                - space_dedupe
                - trim
                - object_field_set:
                    object: product
                    field: description
        - find:
            path: .product-style
            do:
            - parse
            - space_dedupe
            - trim
            - object_field_set:
                object: product
                field: sku
        - find:
            path: .price
            do:
            - find:
                path: .red-text
                do:
                - parse:
                    filter:
                        - (\d+\.?\d*)
                - if:
                    match: (\d)
                    do:
                    - object_field_set:
                        object: product
                        field: price
                        type: float
                    - register_set: USD
                    - object_field_set:
                        object: product
                        field: currency
                    - register_set: 1
                    - variable_set: stp
            - find:
                path: span[data-test="test"]
                do:
                - variable_get: stp
                - if:
                    match: (1)
                    else:
                    - parse:
                        filter:
                            - (\d+\.?\d*)
                    - if:
                        match: (\d)
                        do:
                        - object_field_set:
                            object: product
                            field: price
                            type: float
                        - register_set: USD
                        - object_field_set:
                            object: product
                            field: currency
        - find:
            path: .product-name
            do:
            - parse
            - space_dedupe
            - trim
            - object_field_set:
                object: product
                field: name
        - find:
            path: .main-img
            do:
            - parse:
                attr: src
            - object_field_set:
                object: product
                field: images
                joinby: "|"
        - find:
            path: .logo
            slice: 0
            do:
            - parse
            - space_dedupe
            - trim
            - object_field_set:
                object: product
                field: brand
        - find:
            path: .breadcrumbs a
            do:
            - parse
            - space_dedupe
            - trim
            - object_field_set:
                object: product
                field: category
                joinby: "|"
        - find:
            path: '.product-details > input#skuVarData'
            do:
            - parse:
                attr: value
            - normalize:
                routine: replace_substring
                args:
                    \s+\/\s+: _
            - normalize:
                routine: json2xml
            - to_block
            - find:
                path: body_safe > name
                do:
                - parse
                - space_dedupe
                - trim
                - object_field_set:
                    object: product
                    field: name
            - find:
                path: colors
                do:
                - find:
                    path: zoomimage
                    do:
                    - parse:
                        filter:
                            - \s*(.+)\?
                    - variable_set: imgxl
                    - register_set: <%imgxl%>?$ProductZoom$
                    - object_field_set:
                        object: product
                        field: images
                        joinby: "|"
                - find:
                    path: name
                    do:
                    - parse
                    - space_dedupe
                    - trim
                    - object_field_set:
                        object: product
                        field: variations
                        joinby: "|"
        - object_save:
            name: product