Directive unit-testing document mouse listeners

In a directive, I am binding a listener to mouseup event on $document imported using dependency-injection.

It works well in the browser but when I try to unit test it, I come to some problem.

Basically, my directive is like this:

angular.module('test', [])
.directive('myDirective', function($document) {
    return {
        restrict: 'A',
        link: function(scope, element) {
            $document.on('mouseup', scope.mouseUp);

            scope.mouseUp = function() {
                console.log('mouse up')
            };
        }
    };
});

and my test:

var element, scope, isolateScope, document;

beforeEach(inject(function($rootScope, $compile, $document) {
    scope = $rootScope.$new();
    document = $document;
    element = angular.element('<div my-directive"></div>');
    element = $compile(element)(scope);
    angular.element(document).find('body').append(element);
    scope.$digest();
    isolateScope = element.isolateScope();
}));

it('should react correctly to mousedown events', function() {
    spyOn(isolateScope, 'mouseUp');
    document.trigger('mouseup');
    scope.$digest();
    expect(isolateScope.mouseUp).toHaveBeenCalled();
});

It seems that the $document I use in the directive is different that the one I use in the test...