Angular Directive not for child element

I have a directive which i want to add to a div but not to its child divs. I want a div to be draggable so that this div is draggable and its child divs are dragged with it. But i don't want the child divs to be able to drag its parent div.

Here is the HTML template

<div div-draggable id="chColorPickerWrapper">
    <div>
        <div id="colorDiv"> </div>    
        <div id="whiteToTransparent"> </div>
        <div id="blackToTransparent"> </div>
    </div>
</div>

and here is the directive

chDirectives.directive('divDraggable',['$document', function($document){
    return{
        restrict: 'A',
        link : link
    };

    function link(scope,element,attrs)
    {
        var startX = 0;
        var startY = 0;
        var x = 0;
        var y = 0;

        var mouseMove = function(event) {
            event.preventDefault();
            y = event.pageY - startY;
            x = event.pageX - startX;

            element.css({
                top: y + 'px',
                left:  x + 'px'
            });
        }

        var mouseUp = function() {
            console.log('UNBIND');
            $document.off('mousemove', mouseMove);
            $document.off('mouseup', mouseUp);
        }

        var mouseDown = function(event){
            console.log('mouseDown');
            event.preventDefault();
            startX = event.pageX - x;
            startY = event.pageY - y;
            $document.on('mousemove', mouseMove);
            $document.on('mouseup', mouseUp);
        }


        element.bind('mousedown',mouseDown);
        element.css({
            position: 'relative',
            cursor: 'pointer',
            display: 'block',
        });        
    }
}]);

When i click on the colorDiv i don't want it to be draggable. How can i overwrite or erase this behavior for the child divs ?

if you have any suggestions to make my code better i would appreciate that too, because i am fairly new to angularjs

EDIT: here is a example: http://plnkr.co/edit/K52DDXQRCME7S3oMimoC