Modified VisibilityFilter to handle multiple Authorizations objects

core/src/main/java/org/apache/accumulo/core/clientImpl/access/BytesAccess.java

@@ -20,6 +20,8 @@
 
 import static java.nio.charset.StandardCharsets.ISO_8859_1;
 
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
@@ -93,6 +95,19 @@ public boolean canAccess(byte[] expression) {
     }
   }
 
+  public static BytesEvaluator newEvaluator(Collection<Authorizations> authsSet) {
+    Collection<Set<String>> convertedAuths = new ArrayList<>();
+    for (Authorizations auths : authsSet) {
+      List<byte[]> bytesAuths = auths.getAuthorizations();
+      Set<String> stringAuths = new HashSet<>(bytesAuths.size());
+      for (var auth : bytesAuths) {
+        stringAuths.add(new String(auth, ISO_8859_1));
+      }
+      convertedAuths.add(stringAuths);
+    }
+    return new BytesEvaluator(ACCESS.newEvaluator(convertedAuths));
+  }
+
   public static BytesEvaluator newEvaluator(Authorizations auths) {
     List<byte[]> bytesAuths = auths.getAuthorizations();
     Set<String> stringAuths = new HashSet<>(bytesAuths.size());

core/src/main/java/org/apache/accumulo/core/iterators/user/VisibilityFilter.java

@@ -21,6 +21,8 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Map;
 
 import org.apache.accumulo.access.InvalidAccessExpressionException;
@@ -44,12 +46,14 @@
  */
 public class VisibilityFilter extends Filter implements OptionDescriber {
 
+  private static final Logger log = LoggerFactory.getLogger(VisibilityFilter.class);
+  private static final BytesAccess.BytesEvaluator EMPTY_EVALUATOR =
+      BytesAccess.newEvaluator(Authorizations.EMPTY);
+
   private BytesAccess.BytesEvaluator accessEvaluator;
   protected Map<ByteSequence,Boolean> cache;
   private final ArrayByteSequence testVis = new ArrayByteSequence(new byte[0]);
 
-  private static final Logger log = LoggerFactory.getLogger(VisibilityFilter.class);
-
   private static final String AUTHS = "auths";
   private static final String FILTER_INVALID_ONLY = "filterInvalid";
 
@@ -64,10 +68,31 @@ public void init(SortedKeyValueIterator<Key,Value> source, Map<String,String> op
 
     if (!filterInvalid) {
       String auths = options.get(AUTHS);
-      Authorizations authObj = auths == null || auths.isEmpty() ? new Authorizations()
-          : new Authorizations(auths.getBytes(UTF_8));
-
-      this.accessEvaluator = BytesAccess.newEvaluator(authObj);
+      if (auths == null || auths.isEmpty()) {
+        this.accessEvaluator = EMPTY_EVALUATOR;
+      } else if (!auths.startsWith(Authorizations.HEADER)) {
+        // the old serialization format does not support multiple auth sets, so treat the whole
+        // thing as one set
+        this.accessEvaluator = BytesAccess.newEvaluator(new Authorizations(auths.getBytes(UTF_8)));
+      } else {
+        String[] authParts = auths.split(Authorizations.HEADER);
+        if (authParts.length == 0) {
+          this.accessEvaluator = EMPTY_EVALUATOR;
+        } else {
+          Collection<Authorizations> authSet = new ArrayList<>();
+          for (int i = 0; i < authParts.length; i++) {
+            String part = authParts[i];
+            if (part.isEmpty()) {
+              continue;
+            } else {
+              // split removes the HEADER, need to add it back
+              String serializedAuthString = Authorizations.HEADER + authParts[i];
+              authSet.add(new Authorizations(serializedAuthString.getBytes(UTF_8)));
+            }
+          }
+          this.accessEvaluator = BytesAccess.newEvaluator(authSet);
+        }
+      }
     }
     this.cache = new LRUMap<>(1000);
   }
@@ -132,20 +157,28 @@ public IteratorOptions describeOptions() {
     IteratorOptions io = super.describeOptions();
     io.setName("visibilityFilter");
     io.setDescription("The VisibilityFilter allows you to filter for key/value"
-        + " pairs by a set of authorizations or filter invalid labels from corrupt files.");
+        + " pairs by a set(s) of authorizations or filter invalid labels from corrupt files.");
     io.addNamedOption(FILTER_INVALID_ONLY,
         "if 'true', the iterator is instructed to ignore the authorizations and"
             + " only filter invalid visibility labels (default: false)");
-    io.addNamedOption(AUTHS,
-        "the serialized set of authorizations to filter against (default: empty"
-            + " string, accepts only entries visible by all)");
+    io.addNamedOption(AUTHS, "The concatenated serialized set(s) of Authorizations used to "
+        + "filter out Key-Value pairs. Default value is an empty string which allows all "
+        + "pairs to pass through the filter. The value can be set programmatically using "
+        + "the setAuthorizations method or manually using the output of the Authorizations.serialize() "
+        + "method (concatenated with no separator if multiple Authorizations).");
     return io;
   }
 
   public static void setAuthorizations(IteratorSetting setting, Authorizations auths) {
     setting.addOption(AUTHS, auths.serialize());
   }
 
+  public static void setAuthorizations(IteratorSetting setting, Collection<Authorizations> auths) {
+    StringBuilder builder = new StringBuilder();
+    auths.forEach(a -> builder.append(a.serialize()));
+    setting.addOption(AUTHS, builder.toString());
+  }
+
   public static void filterInvalidLabelsOnly(IteratorSetting setting, boolean featureEnabled) {
     setting.addOption(FILTER_INVALID_ONLY, Boolean.toString(featureEnabled));
   }

core/src/test/java/org/apache/accumulo/core/iterators/user/VisibilityFilterTest.java

@@ -25,6 +25,7 @@
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.TreeMap;
@@ -155,6 +156,22 @@ public void testAllowAuthorizedLabelsOnly() throws IOException {
     verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000);
   }
 
+  @Test
+  public void testMulitAllowAuthorizedLabelsOnly() throws IOException {
+    IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
+    VisibilityFilter.setAuthorizations(is,
+        List.of(new Authorizations("abc"), new Authorizations("def")));
+
+    TreeMap<Key,Value> source = createSourceWithHiddenData(1, 2);
+    verify(source, 3, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1);
+
+    source = createSourceWithHiddenData(30, 500);
+    verify(source, 530, is.getOptions(), GOOD, GOOD, GOOD_VIS, 30);
+
+    source = createSourceWithHiddenData(1000, 500);
+    verify(source, 1500, is.getOptions(), GOOD, GOOD, GOOD_VIS, 1000);
+  }
+
   @Test
   public void testAllowUnauthorizedLabelsOnly() throws IOException {
     IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
@@ -171,6 +188,23 @@ public void testAllowUnauthorizedLabelsOnly() throws IOException {
     verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500);
   }
 
+  @Test
+  public void testMultiAllowUnauthorizedLabelsOnly() throws IOException {
+    IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
+    VisibilityFilter.setNegate(is, true);
+    VisibilityFilter.setAuthorizations(is,
+        List.of(new Authorizations("abc"), new Authorizations("def")));
+
+    TreeMap<Key,Value> source = createSourceWithHiddenData(1, 2);
+    verify(source, 3, is.getOptions(), BAD, BAD, HIDDEN_VIS, 2);
+
+    source = createSourceWithHiddenData(30, 500);
+    verify(source, 530, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500);
+
+    source = createSourceWithHiddenData(1000, 500);
+    verify(source, 1500, is.getOptions(), BAD, BAD, HIDDEN_VIS, 500);
+  }
+
   @Test
   public void testNoLabels() throws IOException {
     IteratorSetting is = new IteratorSetting(1, VisibilityFilter.class);
@@ -255,5 +289,4 @@ public void testDeepCopyAfterInit() throws IOException {
     Key k = new Key("row", "cf", "cq", "abc");
     assertTrue(copyFilter.accept(k, new Value()));
   }
-
 }

test/src/main/java/org/apache/accumulo/test/functional/VisibilityIT.java

@@ -37,12 +37,14 @@
 import org.apache.accumulo.core.client.AccumuloClient;
 import org.apache.accumulo.core.client.BatchScanner;
 import org.apache.accumulo.core.client.BatchWriter;
+import org.apache.accumulo.core.client.IteratorSetting;
 import org.apache.accumulo.core.client.Scanner;
 import org.apache.accumulo.core.conf.Property;
 import org.apache.accumulo.core.data.Key;
 import org.apache.accumulo.core.data.Mutation;
 import org.apache.accumulo.core.data.Range;
 import org.apache.accumulo.core.data.Value;
+import org.apache.accumulo.core.iterators.user.VisibilityFilter;
 import org.apache.accumulo.core.security.Authorizations;
 import org.apache.accumulo.core.security.ColumnVisibility;
 import org.apache.accumulo.core.util.ByteArraySet;
@@ -92,6 +94,7 @@ public void run() throws Exception {
 
       insertData(c, table);
       queryData(c, table);
+      queryDataMultiAuth(c, table);
       deleteData(c, table);
 
       insertDefaultData(c, table2);
@@ -222,6 +225,48 @@ private void queryData(AccumuloClient c, String tableName) throws Exception {
     queryData(c, tableName, nss("A", "B", "FOO", "L", "M", "Z"), nss(), expected);
   }
 
+  /**
+   * Configures Scanners with the users default authorizations, then it adds a
+   * MultiAuthVisibilityFilter with different sets of Authorizations
+   */
+  private void queryDataMultiAuth(AccumuloClient c, String tableName) throws Exception {
+
+    c.securityOperations().changeUserAuthorizations(getAdminPrincipal(),
+        new Authorizations("A", "B", "FOO", "L", "M", "Z"));
+
+    Authorizations userAuths = c.securityOperations().getUserAuthorizations(c.whoami());
+
+    Set<String> expectedUserAuths =
+        Set.of("v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", "v10", "v11", "v12", "v13");
+    try (Scanner scanner = c.createScanner(tableName, userAuths);
+        BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) {
+      verify(scanner.iterator(), expectedUserAuths.toArray(new String[] {}));
+
+      bs.setRanges(Collections.singleton(new Range()));
+      verify(bs.iterator(), expectedUserAuths.toArray(new String[] {}));
+    }
+
+    Authorizations entity1 = new Authorizations("A", "B", "FOO", "L", "M");
+    Authorizations entity2 = new Authorizations("B", "FOO", "Z");
+    // should only see entries with no column visibility, B and/or FOO
+    Set<String> expectedAuths = Set.of("v1", "v3", "v11");
+
+    IteratorSetting is = new IteratorSetting(100, "userAuths", VisibilityFilter.class);
+    VisibilityFilter.setAuthorizations(is, Set.of(entity1, entity2));
+
+    try (Scanner scanner = c.createScanner(tableName, userAuths);
+        BatchScanner bs = c.createBatchScanner(tableName, userAuths, 3)) {
+
+      scanner.addScanIterator(is);
+      verify(scanner.iterator(), expectedAuths.toArray(new String[] {}));
+
+      bs.setRanges(Collections.singleton(new Range()));
+      bs.addScanIterator(is);
+      verify(bs.iterator(), expectedAuths.toArray(new String[] {}));
+    }
+
+  }
+
   private void queryData(AccumuloClient c, String tableName, Set<String> allAuths,
       Set<String> userAuths, Map<Set<String>,Set<String>> expected) throws Exception {