.image_holder {
display: block;
height: 100%;
}

.section{
position: relative;
height: 80vh;
background: rgba(255, 226, 57, 0.0);
overflow-y: scroll;
margin: 0 auto;
border: #000000 solid thin;
}

<div class="page_scroller">
<ul >
<li class="list-unstyled col-lg-6 bg-primary"><a href="#" id="prev" class="arrow-up prev">Up</a></li>
<li class="list-unstyled col-lg-6 bg-primary"><a href="#" id="next" class="arrow-down next">Down</a></li>
</ul>
</div>

<div class="section" >

        <div id="highlight-1" class="current image_holder">A</div>
        <div id="highlight-2" class="image_holder ">B</div>
        <div id="highlight-3" class="image_holder ">C</div>
        <div id="highlight-4" class="image_holder ">D</div>

</div>

var scrollTo = function(element) {
    console.log(element);
    $('.section').animate({
        scrollTop: element.offset().top
    }, 500);
}

$('.next').click(function(event) {
    event.preventDefault();
    var $current = $('.section > .image_holder.current');
    var $next = $current.next().first();
    if ($next.length!=0) {
        $current.removeClass('current')
        $next.addClass('current');
        scrollTo($next);
    }
});

$('.prev').click(function(event) {
    event.preventDefault();
    var $current = $('.section > .image_holder.current');
    var $prev = $current.prev().first();
    if ($prev.length!=0) {
        $current.removeClass('current')
        $prev.addClass('current');
        scrollTo($prev);
    }
});

$('.section').animate({
    scrollTop: element.offset().top
}, 500);

$('.section').animate({
    scrollTop: element.position().top
}, 500);