To have validation the way you are requesting, you have to use an ng-model to put your array of colors into the form so that the array can be validated.

Here's a quick example in a plunker where I push a validator on the $parsers pipeline of the ngModelController which will check the colors array length. Keeping the colorRequired directive separate will allow you to have situations where a color is not required. You could also add to that directive so it will take a boolean argument on the attribute so you can decide at run-time if a color should be required.

http://plnkr.co/edit/yFuSXxacSW811WfZqaPC

var app = angular.module('plunker', []);



app.controller('MainCtrl', function($scope) {

  $scope.colors = ['red', 'blue', 'green'];

});



app.directive(colorGroup, [function() {

    use strict;

    return {

        restrict: 'E',

        require: 'ngModel',

        template: '<ng-form name=userForm>

               <ul>

               <li ng-repeat=color in model>

              <input type=text ng-model=color />

              <a href= ng-click=delete($index)>remove</a>

              </li>

              </ul>

              </ng-form>',

        link: function($scope, element, attrs, ngModelCtrl)

        {

            $scope.$watch(function(){

                return ngModelCtrl.$modelValue;

            }, function(){

                $scope.model = ngModelCtrl.$viewValue;

            });



            $scope.delete = function(idx)

            {

                        ngModelCtrl.$viewValue.splice(idx, 1);

                        ngModelCtrl.$setViewValue(ngModelCtrl.$viewValue);

                        $scope.model = ngModelCtrl.$viewValue;

            }

        }

    }

}]);



app.directive(colorRequired, function() {

    use strict;

    return {

        restrict: 'A',

        require: 'ngModel',

        link: function($scope, element, attrs, ngModelCtrl)

        {

                ngModelCtrl.$setValidity('colorrequired', (ngModelCtrl.$viewValue.length > 0));



                ngModelCtrl.$parsers.push(function(viewValue) {

                    var valid = viewValue.length > 0;

                    ngModelCtrl.$setValidity('colorrequired', valid);

                    return valid ? viewValue : undefined;

                });

            }

        }

    });