From d2355a18b0e73850385f044d806a450b76c1bc38 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 3 Jun 2024 12:38:34 -0700 Subject: [PATCH 01/33] Copy the suppress warning annotations to autovalue gened celexpr PiperOrigin-RevId: 639877597 --- common/src/main/java/dev/cel/common/ast/CelExpr.java | 1 + 1 file changed, 1 insertion(+) diff --git a/common/src/main/java/dev/cel/common/ast/CelExpr.java b/common/src/main/java/dev/cel/common/ast/CelExpr.java index 6af628551..4c08a6d05 100644 --- a/common/src/main/java/dev/cel/common/ast/CelExpr.java +++ b/common/src/main/java/dev/cel/common/ast/CelExpr.java @@ -34,6 +34,7 @@ *

This is the native type equivalent of Expr message in syntax.proto. */ @AutoValue +@AutoValue.CopyAnnotations @Immutable @SuppressWarnings("unchecked") // Class ensures only the super type is used public abstract class CelExpr implements Expression { From 293fb2b9150b231f2958b3734b86d9e81b578f17 Mon Sep 17 00:00:00 2001 From: CEL Dev Team Date: Tue, 18 Jun 2024 15:00:58 -0700 Subject: [PATCH 02/33] Implement CEL Set Extension Implement `.contains`, `intersects` and `.equivalent` for Sets. PiperOrigin-RevId: 644521835 --- extensions/BUILD.bazel | 5 + .../main/java/dev/cel/extensions/BUILD.bazel | 18 + .../dev/cel/extensions/CelExtensions.java | 38 +- .../dev/cel/extensions/CelSetsExtensions.java | 240 +++++++++++++ .../main/java/dev/cel/extensions/README.md | 68 ++++ .../test/java/dev/cel/extensions/BUILD.bazel | 1 + .../cel/extensions/CelSetsExtensionsTest.java | 328 ++++++++++++++++++ 7 files changed, 697 insertions(+), 1 deletion(-) create mode 100644 extensions/src/main/java/dev/cel/extensions/CelSetsExtensions.java create mode 100644 extensions/src/test/java/dev/cel/extensions/CelSetsExtensionsTest.java diff --git a/extensions/BUILD.bazel b/extensions/BUILD.bazel index ef0e9c9fd..70956e336 100644 --- a/extensions/BUILD.bazel +++ b/extensions/BUILD.bazel @@ -22,3 +22,8 @@ java_library( name = "optional_library", exports = ["//extensions/src/main/java/dev/cel/extensions:optional_library"], ) + +java_library( + name = "sets", + exports = ["//extensions/src/main/java/dev/cel/extensions:sets"], +) diff --git a/extensions/src/main/java/dev/cel/extensions/BUILD.bazel b/extensions/src/main/java/dev/cel/extensions/BUILD.bazel index bf42c2ae8..da8aba9a0 100644 --- a/extensions/src/main/java/dev/cel/extensions/BUILD.bazel +++ b/extensions/src/main/java/dev/cel/extensions/BUILD.bazel @@ -18,6 +18,7 @@ java_library( ":encoders", ":math", ":protos", + ":sets", ":strings", "//common:options", "@maven//:com_google_guava_guava", @@ -126,3 +127,20 @@ java_library( "@maven//:com_google_protobuf_protobuf_java", ], ) + +java_library( + name = "sets", + srcs = ["CelSetsExtensions.java"], + tags = [ + ], + deps = [ + "//checker:checker_builder", + "//common:compiler_common", + "//common/internal:comparison_functions", + "//common/types", + "//compiler:compiler_builder", + "//runtime", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) diff --git a/extensions/src/main/java/dev/cel/extensions/CelExtensions.java b/extensions/src/main/java/dev/cel/extensions/CelExtensions.java index c3d9fdbbf..5515d6a89 100644 --- a/extensions/src/main/java/dev/cel/extensions/CelExtensions.java +++ b/extensions/src/main/java/dev/cel/extensions/CelExtensions.java @@ -1,4 +1,4 @@ -// Copyright 2022 Google LLC +// Copyright 2024 Google LLC // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -30,6 +30,7 @@ public final class CelExtensions { private static final CelProtoExtensions PROTO_EXTENSIONS = new CelProtoExtensions(); private static final CelBindingsExtensions BINDINGS_EXTENSIONS = new CelBindingsExtensions(); private static final CelEncoderExtensions ENCODER_EXTENSIONS = new CelEncoderExtensions(); + private static final CelSetsExtensions SET_EXTENSIONS = new CelSetsExtensions(); /** * Extended functions for string manipulation. @@ -170,5 +171,40 @@ public static CelEncoderExtensions encoders() { return ENCODER_EXTENSIONS; } + /** + * Extended functions for Set manipulation. + * + *

Refer to README.md for available functions. + * + *

This will include all functions denoted in {@link CelSetExtensions.Function}, including any + * future additions. To expose only a subset of functions, use {@link + * #sets(CelSetExtensions.Function...)} instead. + */ + public static CelSetsExtensions sets() { + return SET_EXTENSIONS; + } + + /** + * Extended functions for Set manipulation. + * + *

Refer to README.md for available functions. + * + *

This will include only the specific functions denoted by {@link CelSetsExtensions.Function}. + */ + public static CelSetsExtensions sets(CelSetsExtensions.Function... functions) { + return sets(ImmutableSet.copyOf(functions)); + } + + /** + * Extended functions for Set manipulation. + * + *

Refer to README.md for available functions. + * + *

This will include only the specific functions denoted by {@link CelSetsExtensions.Function}. + */ + public static CelSetsExtensions sets(Set functions) { + return new CelSetsExtensions(functions); + } + private CelExtensions() {} } diff --git a/extensions/src/main/java/dev/cel/extensions/CelSetsExtensions.java b/extensions/src/main/java/dev/cel/extensions/CelSetsExtensions.java new file mode 100644 index 000000000..10fcbdd0c --- /dev/null +++ b/extensions/src/main/java/dev/cel/extensions/CelSetsExtensions.java @@ -0,0 +1,240 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.extensions; + +import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.Immutable; +import dev.cel.checker.CelCheckerBuilder; +import dev.cel.common.CelFunctionDecl; +import dev.cel.common.CelOverloadDecl; +import dev.cel.common.internal.ComparisonFunctions; +import dev.cel.common.types.ListType; +import dev.cel.common.types.SimpleType; +import dev.cel.common.types.TypeParamType; +import dev.cel.compiler.CelCompilerLibrary; +import dev.cel.runtime.CelRuntime; +import dev.cel.runtime.CelRuntimeBuilder; +import dev.cel.runtime.CelRuntimeLibrary; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Set; + +/** + * Internal implementation of CEL Set extensions. + * + *

TODO: https://github.com/google/cel-go/blob/master/ext/sets.go#L127 + * + *

Invoking in operator will result in O(n) complexity. We need to wire in the CEL optimizers to + * rewrite the AST into a map to achieve a O(1) lookup. + */ +@Immutable +@SuppressWarnings({"unchecked"}) // Unchecked: Type-checker guarantees casting safety. +public final class CelSetsExtensions implements CelCompilerLibrary, CelRuntimeLibrary { + + private static final String SET_CONTAINS_FUNCTION = "sets.contains"; + private static final String SET_CONTAINS_OVERLOAD_DOC = + "Returns whether the first list argument contains all elements in the second list" + + " argument. The list may contain elements of any type and standard CEL" + + " equality is used to determine whether a value exists in both lists. If the" + + " second list is empty, the result will always return true."; + private static final String SET_EQUIVALENT_FUNCTION = "sets.equivalent"; + private static final String SET_EQUIVALENT_OVERLOAD_DOC = + "Returns whether the first and second list are set equivalent. Lists are set equivalent if" + + " for every item in the first list, there is an element in the second which is equal." + + " The lists may not be of the same size as they do not guarantee the elements within" + + " them are unique, so size does not factor intothe computation."; + private static final String SET_INTERSECTS_FUNCTION = "sets.intersects"; + private static final String SET_INTERSECTS_OVERLOAD_DOC = + "Returns whether the first and second list intersect. Lists intersect if there is at least" + + " one element in the first list which is equal to an element in the second list. The" + + " lists may not be of the same size as they do not guarantee the elements within them" + + " are unique, so size does not factor into the computation. If either list is empty," + + " the result will be false."; + + /** Denotes the set extension function. */ + public enum Function { + CONTAINS( + CelFunctionDecl.newFunctionDeclaration( + SET_CONTAINS_FUNCTION, + CelOverloadDecl.newGlobalOverload( + "list_sets_contains_list", + SET_CONTAINS_OVERLOAD_DOC, + SimpleType.BOOL, + ListType.create(TypeParamType.create("T")), + ListType.create(TypeParamType.create("T")))), + CelRuntime.CelFunctionBinding.from( + "list_sets_contains_list", + Collection.class, + Collection.class, + CelSetsExtensions::containsAll)), + EQUIVALENT( + CelFunctionDecl.newFunctionDeclaration( + SET_EQUIVALENT_FUNCTION, + CelOverloadDecl.newGlobalOverload( + "list_sets_equivalent_list", + SET_EQUIVALENT_OVERLOAD_DOC, + SimpleType.BOOL, + ListType.create(TypeParamType.create("T")), + ListType.create(TypeParamType.create("T")))), + CelRuntime.CelFunctionBinding.from( + "list_sets_equivalent_list", + Collection.class, + Collection.class, + (listA, listB) -> containsAll(listA, listB) && containsAll(listB, listA))), + INTERSECTS( + CelFunctionDecl.newFunctionDeclaration( + SET_INTERSECTS_FUNCTION, + CelOverloadDecl.newGlobalOverload( + "list_sets_intersects_list", + SET_INTERSECTS_OVERLOAD_DOC, + SimpleType.BOOL, + ListType.create(TypeParamType.create("T")), + ListType.create(TypeParamType.create("T")))), + CelRuntime.CelFunctionBinding.from( + "list_sets_intersects_list", + Collection.class, + Collection.class, + CelSetsExtensions::setIntersects)); + + private final CelFunctionDecl functionDecl; + private final ImmutableSet functionBindings; + + Function(CelFunctionDecl functionDecl, CelRuntime.CelFunctionBinding... functionBindings) { + this.functionDecl = functionDecl; + this.functionBindings = ImmutableSet.copyOf(functionBindings); + } + } + + private final ImmutableSet functions; + + CelSetsExtensions() { + this(ImmutableSet.copyOf(Function.values())); + } + + CelSetsExtensions(Set functions) { + this.functions = ImmutableSet.copyOf(functions); + } + + @Override + public void setCheckerOptions(CelCheckerBuilder checkerBuilder) { + functions.forEach(function -> checkerBuilder.addFunctionDeclarations(function.functionDecl)); + } + + @Override + public void setRuntimeOptions(CelRuntimeBuilder runtimeBuilder) { + functions.forEach(function -> runtimeBuilder.addFunctionBindings(function.functionBindings)); + } + + /** + * This implementation iterates over the specified collection, checking each element returned by + * the iterator in turn to see if it's contained in this collection. If all elements are so + * contained true is returned, otherwise false. + * + *

This is picked verbatim as implemented in the Java standard library + * Collections.containsAll() method. + * + * @see #contains(Object) + */ + private static boolean containsAll(Collection list, Collection subList) { + for (T e : subList) { + if (!contains(e, list)) { + return false; + } + } + return true; + } + + /** + * This implementation iterates over the elements in the collection, checking each element in turn + * for equality with the specified element. + * + *

This is picked verbatim as implemented in the Java standard library Collections.contains() + * method. + * + *

Source: + * https://hg.openjdk.org/jdk8u/jdk8u-dev/jdk/file/c5d02f908fb2/src/share/classes/java/util/AbstractCollection.java#l98 + */ + private static boolean contains(Object o, Collection list) { + Iterator it = list.iterator(); + if (o == null) { + while (it.hasNext()) { + if (it.next() == null) { + return true; + } + } + } else { + while (it.hasNext()) { + Object item = it.next(); + if (objectsEquals(item, o)) { // TODO: Support Maps. + return true; + } + } + } + return false; + } + + private static boolean objectsEquals(Object o1, Object o2) { + if (o1 == o2) { + return true; + } + if (o1 == null || o2 == null) { + return false; + } + if (isNumeric(o1) && isNumeric(o2)) { + if (o1.getClass().equals(o2.getClass())) { + return o1.equals(o2); + } + return ComparisonFunctions.numericEquals((Number) o1, (Number) o2); + } + if (isList(o1) && isList(o2)) { + Collection list1 = (Collection) o1; + Collection list2 = (Collection) o2; + if (list1.size() != list2.size()) { + return false; + } + Iterator iterator1 = list1.iterator(); + Iterator iterator2 = list2.iterator(); + boolean result = true; + while (iterator1.hasNext() && iterator2.hasNext()) { + Object p1 = iterator1.next(); + Object p2 = iterator2.next(); + result = result && objectsEquals(p1, p2); + } + return result; + } + return o1.equals(o2); + } + + private static boolean isNumeric(Object o) { + return o instanceof Number; + } + + private static boolean isList(Object o) { + return o instanceof List; + } + + private static boolean setIntersects(Collection listA, Collection listB) { + if (listA.isEmpty() || listB.isEmpty()) { + return false; + } + for (T element : listB) { + if (contains(element, listA)) { + return true; + } + } + return false; + } +} diff --git a/extensions/src/main/java/dev/cel/extensions/README.md b/extensions/src/main/java/dev/cel/extensions/README.md index 1c78f0364..1163f334c 100644 --- a/extensions/src/main/java/dev/cel/extensions/README.md +++ b/extensions/src/main/java/dev/cel/extensions/README.md @@ -338,3 +338,71 @@ Example: base64.encode(b'hello') // return 'aGVsbG8=' +## Sets + +Sets provides set relationship tests. + +There is no set type within CEL, and while one may be introduced in the future, +there are cases where a `list` type is known to behave like a set. For such +cases, this library provides some basic functionality for determining set +containment, equivalence, and intersection. + +### Sets.Contains + +Returns whether the first list argument contains all elements in the second list +argument. The list may contain elements of any type and standard CEL equality is +used to determine whether a value exists in both lists. If the second list is +empty, the result will always return true. + +``` +sets.contains(list(T), list(T)) -> bool +``` + +Examples: + +``` +sets.contains([], []) // true +sets.contains([], [1]) // false +sets.contains([1, 2, 3, 4], [2, 3]) // true +sets.contains([1, 2.0, 3u], [1.0, 2u, 3]) // true +``` + +### Sets.Equivalent + +Returns whether the first and second list are set equivalent. Lists are set +equivalent if for every item in the first list, there is an element in the +second which is equal. The lists may not be of the same size as they do not +guarantee the elements within them are unique, so size does not factor into the +computation. + +``` +sets.equivalent(list(T), list(T)) -> bool +``` + +Examples: + +``` +sets.equivalent([], []) // true +sets.equivalent([1], [1, 1]) // true +sets.equivalent([1], [1u, 1.0]) // true +sets.equivalent([1, 2, 3], [3u, 2.0, 1]) // true +``` + +### Sets.Intersects + +Returns whether the first list has at least one element whose value is equal to +an element in the second list. If either list is empty, the result will be +false. + +``` +sets.intersects(list(T), list(T)) -> bool +``` + +Examples: + +``` +sets.intersects([1], []) // false +sets.intersects([1], [1, 2]) // true +sets.intersects([[1], [2, 3]], [[1, 2], [2, 3.0]]) // true +``` + diff --git a/extensions/src/test/java/dev/cel/extensions/BUILD.bazel b/extensions/src/test/java/dev/cel/extensions/BUILD.bazel index c2a362a81..1edea7cb0 100644 --- a/extensions/src/test/java/dev/cel/extensions/BUILD.bazel +++ b/extensions/src/test/java/dev/cel/extensions/BUILD.bazel @@ -23,6 +23,7 @@ java_library( "//extensions", "//extensions:math", "//extensions:optional_library", + "//extensions:sets", "//extensions:strings", "//parser:macro", "//runtime", diff --git a/extensions/src/test/java/dev/cel/extensions/CelSetsExtensionsTest.java b/extensions/src/test/java/dev/cel/extensions/CelSetsExtensionsTest.java new file mode 100644 index 000000000..70396ee45 --- /dev/null +++ b/extensions/src/test/java/dev/cel/extensions/CelSetsExtensionsTest.java @@ -0,0 +1,328 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.extensions; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import com.google.testing.junit.testparameterinjector.TestParameters; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelValidationException; +import dev.cel.common.CelValidationResult; +import dev.cel.common.types.ListType; +import dev.cel.common.types.SimpleType; +import dev.cel.compiler.CelCompiler; +import dev.cel.compiler.CelCompilerFactory; +import dev.cel.extensions.CelSetsExtensions.Function; +import dev.cel.runtime.CelEvaluationException; +import dev.cel.runtime.CelRuntime; +import dev.cel.runtime.CelRuntimeFactory; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public final class CelSetsExtensionsTest { + private static final CelCompiler COMPILER = + CelCompilerFactory.standardCelCompilerBuilder() + .addLibraries(CelExtensions.sets()) + .addVar("list", ListType.create(SimpleType.INT)) + .addVar("subList", ListType.create(SimpleType.INT)) + .build(); + + private static final CelRuntime RUNTIME = + CelRuntimeFactory.standardCelRuntimeBuilder().addLibraries(CelExtensions.sets()).build(); + + @Test + public void contains_integerListWithSameValue_succeeds() throws Exception { + ImmutableList list = ImmutableList.of(1, 2, 3, 4); + ImmutableList subList = ImmutableList.of(1, 2, 3, 4); + CelAbstractSyntaxTree ast = COMPILER.compile("sets.contains(list, subList)").getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(ImmutableMap.of("list", list, "subList", subList)); + + assertThat(result).isEqualTo(true); + } + + @Test + public void contains_integerListAsExpression_succeeds() throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile("sets.contains([1, 1], [1])").getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(); + + assertThat(result).isEqualTo(true); + } + + @Test + @TestParameters("{list: [1, 2, 3, 4], subList: [1, 2, 3, 4], expected: true}") + @TestParameters("{list: [5, 4, 3, 2, 1], subList: [1, 2, 3], expected: true}") + @TestParameters("{list: [5, 4, 3, 2, 1], subList: [1, 1, 1, 1, 1], expected: true}") + @TestParameters("{list: [], subList: [], expected: true}") + @TestParameters("{list: [1], subList: [], expected: true}") + @TestParameters("{list: [], subList: [1], expected: false}") + @TestParameters("{list: [1], subList: [1], expected: true}") + @TestParameters("{list: [1], subList: [1, 1], expected: true}") + @TestParameters("{list: [1, 1], subList: [1, 1], expected: true}") + @TestParameters("{list: [2, 1], subList: [1], expected: true}") + @TestParameters("{list: [1, 2, 3, 4], subList: [2, 3], expected: true}") + @TestParameters("{list: [1], subList: [2], expected: false}") + @TestParameters("{list: [1], subList: [1, 2], expected: false}") + public void contains_withIntTypes_succeeds( + List list, List subList, boolean expected) throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile("sets.contains(list, subList)").getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(ImmutableMap.of("list", list, "subList", subList)); + + assertThat(result).isEqualTo(expected); + } + + @Test + @TestParameters("{list: [1.0], subList: [1.0, 1.0], expected: true}") + @TestParameters("{list: [1.0, 1.00], subList: [1], expected: true}") + @TestParameters("{list: [1.0], subList: [1.00], expected: true}") + @TestParameters("{list: [1.414], subList: [], expected: true}") + @TestParameters("{list: [], subList: [1.414], expected: false}") + @TestParameters("{list: [3.14, 2.71], subList: [2.71], expected: true}") + @TestParameters("{list: [3.9], subList: [3.1], expected: false}") + @TestParameters("{list: [3.2], subList: [3.1], expected: false}") + @TestParameters("{list: [2, 3.0], subList: [2, 3], expected: true}") + public void contains_withDoubleTypes_succeeds( + List list, List subList, boolean expected) throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile("sets.contains(list, subList)").getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(ImmutableMap.of("list", list, "subList", subList)); + + assertThat(result).isEqualTo(expected); + } + + @Test + @TestParameters("{expression: 'sets.contains([[1], [2, 3]], [[2, 3]])', expected: true}") + @TestParameters("{expression: 'sets.contains([[1], [2], [3]], [[2, 3]])', expected: false}") + @TestParameters( + "{expression: 'sets.contains([[1, 2], [2, 3]], [[1], [2, 3.0]])', expected: false}") + @TestParameters("{expression: 'sets.contains([[1], [2, 3.0]], [[2, 3]])', expected: true}") + public void contains_withNestedLists_succeeds(String expression, boolean expected) + throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile(expression).getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(); + + assertThat(result).isEqualTo(expected); + } + + @Test + @TestParameters("{expression: 'sets.contains([1, \"1\"], [1])', expected: true}") + @TestParameters("{expression: 'sets.contains([1], [1, \"1\"])', expected: false}") + public void contains_withMixingIntAndString_succeeds(String expression, boolean expected) + throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile(expression).getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(); + + assertThat(result).isEqualTo(expected); + } + + @Test + @TestParameters("{expression: 'sets.contains([1], [\"1\"])'}") + @TestParameters("{expression: 'sets.contains([\"1\"], [1])'}") + public void contains_withMixingIntAndString_throwsException(String expression) throws Exception { + CelValidationResult invalidData = COMPILER.compile(expression); + + assertThat(invalidData.getErrors()).hasSize(1); + assertThat(invalidData.getErrors().get(0).getMessage()) + .contains("found no matching overload for 'sets.contains'"); + } + + @Test + public void contains_withMixedValues_succeeds() throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile("sets.contains([1, 2], [2u, 2.0])").getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(); + + assertThat(result).isEqualTo(true); + } + + @Test + @TestParameters("{expression: 'sets.contains([[1], [2, 3.0]], [[2, 3]])', expected: true}") + @TestParameters( + "{expression: 'sets.contains([[1], [2, 3.0], [[[[[5]]]]]], [[[[[[5]]]]]])', expected: true}") + @TestParameters( + "{expression: 'sets.contains([[1], [2, 3.0], [[[[[5]]]]]], [[[[[[5, 1]]]]]])', expected:" + + " false}") + @TestParameters( + "{expression: 'sets.contains([[1], [2, 3.0], [[[[[5, 1]]]]]], [[[[[[5]]]]]])', expected:" + + " false}") + @TestParameters( + "{expression: 'sets.contains([[[[[[5]]]]]], [[1], [2, 3.0], [[[[[5]]]]]])', expected: false}") + public void contains_withMultiLevelNestedList_succeeds(String expression, boolean expected) + throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile(expression).getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(); + + assertThat(result).isEqualTo(expected); + } + + @Test + @TestParameters("{expression: 'sets.equivalent([], [])', expected: true}") + @TestParameters("{expression: 'sets.equivalent([1], [1])', expected: true}") + @TestParameters("{expression: 'sets.equivalent([1], [1, 1])', expected: true}") + @TestParameters("{expression: 'sets.equivalent([1, 1], [1])', expected: true}") + @TestParameters("{expression: 'sets.equivalent([[1], [2, 3]], [[1], [2, 3]])', expected: true}") + @TestParameters("{expression: 'sets.equivalent([2, 1], [1])', expected: false}") + @TestParameters("{expression: 'sets.equivalent([1], [1, 2])', expected: false}") + @TestParameters("{expression: 'sets.equivalent([1, 2], [1, 2, 3])', expected: false}") + @TestParameters("{expression: 'sets.equivalent([1, 2], [2, 2, 2])', expected: false}") + public void equivalent_withIntTypes_succeeds(String expression, boolean expected) + throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile(expression).getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(); + + assertThat(result).isEqualTo(expected); + } + + @Test + @TestParameters("{expression: 'sets.equivalent([1, 2, 3], [3u, 2.0, 1])', expected: true}") + @TestParameters("{expression: 'sets.equivalent([1], [1u, 1.0])', expected: true}") + @TestParameters("{expression: 'sets.equivalent([1], [1u, 1.0])', expected: true}") + @TestParameters( + "{expression: 'sets.equivalent([[1.0], [2, 3]], [[1], [2, 3.0]])', expected: true}") + @TestParameters("{expression: 'sets.equivalent([1, 2], [2u, 2, 2.0])', expected: false}") + @TestParameters("{expression: 'sets.equivalent([1, 2], [1u, 2, 2.3])', expected: false}") + public void equivalent_withMixedTypes_succeeds(String expression, boolean expected) + throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile(expression).getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(); + + assertThat(result).isEqualTo(expected); + } + + @Test + @TestParameters("{expression: 'sets.intersects([], [])', expected: false}") + @TestParameters("{expression: 'sets.intersects([1], [])', expected: false}") + @TestParameters("{expression: 'sets.intersects([], [1])', expected: false}") + @TestParameters("{expression: 'sets.intersects([1], [1])', expected: true}") + @TestParameters("{expression: 'sets.intersects([1], [2])', expected: false}") + @TestParameters("{expression: 'sets.intersects([1], [1, 1])', expected: true}") + @TestParameters("{expression: 'sets.intersects([1, 1], [1])', expected: true}") + @TestParameters("{expression: 'sets.intersects([2, 1], [1])', expected: true}") + @TestParameters("{expression: 'sets.intersects([1], [1, 2])', expected: true}") + @TestParameters("{expression: 'sets.intersects([1], [1.0, 2])', expected: true}") + @TestParameters("{expression: 'sets.intersects([1, 2], [2u, 2, 2.0])', expected: true}") + @TestParameters("{expression: 'sets.intersects([1, 2], [1, 2, 2.3])', expected: true}") + @TestParameters("{expression: 'sets.intersects([0, 1, 2], [1, 2, 2.3])', expected: true}") + @TestParameters("{expression: 'sets.intersects([1, 2], [1u, 2, 2.3])', expected: true}") + @TestParameters( + "{expression: 'sets.intersects([[1], [2, 3]], [[1, 2], [2, 3.0]])', expected: true}") + @TestParameters("{expression: 'sets.intersects([1], [\"1\", 2])', expected: false}") + @TestParameters("{expression: 'sets.intersects([1], [1.1, 2])', expected: false}") + @TestParameters("{expression: 'sets.intersects([1], [1.1, 2u])', expected: false}") + public void intersects_withMixedTypes_succeeds(String expression, boolean expected) + throws Exception { + CelAbstractSyntaxTree ast = COMPILER.compile(expression).getAst(); + CelRuntime.Program program = RUNTIME.createProgram(ast); + + Object result = program.eval(); + + assertThat(result).isEqualTo(expected); + } + + @Test + public void setsExtension_containsFunctionSubset_succeeds() throws Exception { + CelSetsExtensions setsExtensions = CelExtensions.sets(Function.CONTAINS); + CelCompiler celCompiler = + CelCompilerFactory.standardCelCompilerBuilder().addLibraries(setsExtensions).build(); + CelRuntime celRuntime = + CelRuntimeFactory.standardCelRuntimeBuilder().addLibraries(setsExtensions).build(); + + Object evaluatedResult = + celRuntime.createProgram(celCompiler.compile("sets.contains([1, 2], [2])").getAst()).eval(); + + assertThat(evaluatedResult).isEqualTo(true); + } + + @Test + public void setsExtension_equivalentFunctionSubset_succeeds() throws Exception { + CelSetsExtensions setsExtensions = CelExtensions.sets(Function.EQUIVALENT); + CelCompiler celCompiler = + CelCompilerFactory.standardCelCompilerBuilder().addLibraries(setsExtensions).build(); + CelRuntime celRuntime = + CelRuntimeFactory.standardCelRuntimeBuilder().addLibraries(setsExtensions).build(); + + Object evaluatedResult = + celRuntime + .createProgram(celCompiler.compile("sets.equivalent([1, 1], [1])").getAst()) + .eval(); + + assertThat(evaluatedResult).isEqualTo(true); + } + + @Test + public void setsExtension_intersectsFunctionSubset_succeeds() throws Exception { + CelSetsExtensions setsExtensions = CelExtensions.sets(Function.INTERSECTS); + CelCompiler celCompiler = + CelCompilerFactory.standardCelCompilerBuilder().addLibraries(setsExtensions).build(); + CelRuntime celRuntime = + CelRuntimeFactory.standardCelRuntimeBuilder().addLibraries(setsExtensions).build(); + + Object evaluatedResult = + celRuntime + .createProgram(celCompiler.compile("sets.intersects([1, 1], [1])").getAst()) + .eval(); + + assertThat(evaluatedResult).isEqualTo(true); + } + + @Test + public void setsExtension_compileUnallowedFunction_throws() { + CelSetsExtensions setsExtensions = CelExtensions.sets(Function.EQUIVALENT); + CelCompiler celCompiler = + CelCompilerFactory.standardCelCompilerBuilder().addLibraries(setsExtensions).build(); + + assertThrows( + CelValidationException.class, + () -> celCompiler.compile("sets.contains([1, 2], [2])").getAst()); + } + + @Test + public void setsExtension_evaluateUnallowedFunction_throws() throws Exception { + CelSetsExtensions setsExtensions = CelExtensions.sets(Function.CONTAINS, Function.EQUIVALENT); + CelCompiler celCompiler = + CelCompilerFactory.standardCelCompilerBuilder().addLibraries(setsExtensions).build(); + CelRuntime celRuntime = + CelRuntimeFactory.standardCelRuntimeBuilder() + .addLibraries(CelExtensions.sets(Function.EQUIVALENT)) + .build(); + + CelAbstractSyntaxTree ast = celCompiler.compile("sets.contains([1, 2], [2])").getAst(); + + assertThrows(CelEvaluationException.class, () -> celRuntime.createProgram(ast).eval()); + } +} From b28987191fe19af33b92b216a8906287d48a3e74 Mon Sep 17 00:00:00 2001 From: Kurt Alfred Kluever Date: Thu, 20 Jun 2024 10:49:35 -0700 Subject: [PATCH 03/33] No public description PiperOrigin-RevId: 645086818 --- WORKSPACE | 2 +- bundle/src/test/java/dev/cel/bundle/CelImplTest.java | 2 +- .../src/main/java/dev/cel/checker/DescriptorTypeProvider.java | 2 +- checker/src/main/java/dev/cel/checker/Env.java | 2 +- checker/src/main/java/dev/cel/checker/ExprChecker.java | 2 +- checker/src/main/java/dev/cel/checker/TypeFormatter.java | 2 +- checker/src/main/java/dev/cel/checker/TypeProvider.java | 2 +- .../src/main/java/dev/cel/checker/TypeProviderLegacyImpl.java | 2 +- checker/src/main/java/dev/cel/checker/Types.java | 2 +- common/src/main/java/dev/cel/common/CelValidationResult.java | 2 +- common/src/main/java/dev/cel/common/internal/AdaptingTypes.java | 2 +- common/src/main/java/dev/cel/common/internal/Errors.java | 2 +- common/src/main/java/dev/cel/common/internal/ProtoAdapter.java | 2 +- common/src/main/java/dev/cel/common/values/ListValue.java | 2 +- common/src/main/java/dev/cel/common/values/MapValue.java | 2 +- common/src/main/java/dev/cel/common/values/OptionalValue.java | 2 +- runtime/src/main/java/dev/cel/runtime/Activation.java | 2 +- runtime/src/main/java/dev/cel/runtime/CelRuntimeLegacyImpl.java | 2 +- .../main/java/dev/cel/runtime/DescriptorMessageProvider.java | 2 +- .../src/main/java/dev/cel/runtime/DynamicMessageFactory.java | 2 +- runtime/src/main/java/dev/cel/runtime/GlobalResolver.java | 2 +- runtime/src/main/java/dev/cel/runtime/InterpreterException.java | 2 +- runtime/src/main/java/dev/cel/runtime/InterpreterUtil.java | 2 +- runtime/src/main/java/dev/cel/runtime/MessageFactory.java | 2 +- .../java/dev/cel/runtime/RuntimeTypeProviderLegacyImpl.java | 2 +- runtime/src/main/java/dev/cel/runtime/StandardTypeResolver.java | 2 +- runtime/src/main/java/dev/cel/runtime/TypeResolver.java | 2 +- runtime/src/test/java/dev/cel/runtime/RuntimeEqualityTest.java | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index e965fca90..cd72fce2c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -73,7 +73,7 @@ maven_install( "com.google.truth.extensions:truth-proto-extension:1.4.2", "com.google.truth:truth:1.4.2", "org.antlr:antlr4-runtime:" + ANTLR4_VERSION, - "org.jspecify:jspecify:0.2.0", + "org.jspecify:jspecify:0.3.0", "org.threeten:threeten-extra:1.8.0", ], repositories = [ diff --git a/bundle/src/test/java/dev/cel/bundle/CelImplTest.java b/bundle/src/test/java/dev/cel/bundle/CelImplTest.java index f1b1dfb00..ef751df18 100644 --- a/bundle/src/test/java/dev/cel/bundle/CelImplTest.java +++ b/bundle/src/test/java/dev/cel/bundle/CelImplTest.java @@ -107,7 +107,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/checker/src/main/java/dev/cel/checker/DescriptorTypeProvider.java b/checker/src/main/java/dev/cel/checker/DescriptorTypeProvider.java index e2fe17351..b5f849d50 100644 --- a/checker/src/main/java/dev/cel/checker/DescriptorTypeProvider.java +++ b/checker/src/main/java/dev/cel/checker/DescriptorTypeProvider.java @@ -41,7 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Set; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * The {@code DescriptorTypeProvider} provides type information for one or more {@link Descriptor} diff --git a/checker/src/main/java/dev/cel/checker/Env.java b/checker/src/main/java/dev/cel/checker/Env.java index e1846f1b8..8cc354fb2 100644 --- a/checker/src/main/java/dev/cel/checker/Env.java +++ b/checker/src/main/java/dev/cel/checker/Env.java @@ -48,7 +48,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * Environment used during checking of expressions. Provides name resolution and error reporting. diff --git a/checker/src/main/java/dev/cel/checker/ExprChecker.java b/checker/src/main/java/dev/cel/checker/ExprChecker.java index 658c7db06..f4f485e08 100644 --- a/checker/src/main/java/dev/cel/checker/ExprChecker.java +++ b/checker/src/main/java/dev/cel/checker/ExprChecker.java @@ -47,7 +47,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * The expression type checker. diff --git a/checker/src/main/java/dev/cel/checker/TypeFormatter.java b/checker/src/main/java/dev/cel/checker/TypeFormatter.java index 450518068..3cdd1a511 100644 --- a/checker/src/main/java/dev/cel/checker/TypeFormatter.java +++ b/checker/src/main/java/dev/cel/checker/TypeFormatter.java @@ -18,7 +18,7 @@ import dev.cel.common.annotations.Internal; import dev.cel.common.types.CelType; import dev.cel.common.types.CelTypes; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * Class to format {@link Type} objects into {@code String} values. diff --git a/checker/src/main/java/dev/cel/checker/TypeProvider.java b/checker/src/main/java/dev/cel/checker/TypeProvider.java index 745789498..10ce2a7d6 100644 --- a/checker/src/main/java/dev/cel/checker/TypeProvider.java +++ b/checker/src/main/java/dev/cel/checker/TypeProvider.java @@ -22,7 +22,7 @@ import dev.cel.common.types.CelTypes; import java.util.Optional; import java.util.function.Function; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * The {@code TypeProvider} defines methods to lookup types and enums, and resolve field types. diff --git a/checker/src/main/java/dev/cel/checker/TypeProviderLegacyImpl.java b/checker/src/main/java/dev/cel/checker/TypeProviderLegacyImpl.java index 497d15698..4d1e0ea32 100644 --- a/checker/src/main/java/dev/cel/checker/TypeProviderLegacyImpl.java +++ b/checker/src/main/java/dev/cel/checker/TypeProviderLegacyImpl.java @@ -26,7 +26,7 @@ import dev.cel.common.types.StructType; import dev.cel.common.types.TypeType; import java.util.Optional; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * The {@code TypeProviderLegacyImpl} acts as a bridge between the old and new type provider APIs diff --git a/checker/src/main/java/dev/cel/checker/Types.java b/checker/src/main/java/dev/cel/checker/Types.java index 5e54dfd6d..7f249fc92 100644 --- a/checker/src/main/java/dev/cel/checker/Types.java +++ b/checker/src/main/java/dev/cel/checker/Types.java @@ -39,7 +39,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * Utilities for dealing with the {@link Type} proto. diff --git a/common/src/main/java/dev/cel/common/CelValidationResult.java b/common/src/main/java/dev/cel/common/CelValidationResult.java index a7e6eceb8..3e52bc8e1 100644 --- a/common/src/main/java/dev/cel/common/CelValidationResult.java +++ b/common/src/main/java/dev/cel/common/CelValidationResult.java @@ -24,7 +24,7 @@ import com.google.errorprone.annotations.Immutable; import com.google.errorprone.annotations.InlineMe; import dev.cel.common.annotations.Internal; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * CelValidationResult encapsulates the {@code CelAbstractSyntaxTree} and {@code CelIssue} set which diff --git a/common/src/main/java/dev/cel/common/internal/AdaptingTypes.java b/common/src/main/java/dev/cel/common/internal/AdaptingTypes.java index 53fe59503..48158bd91 100644 --- a/common/src/main/java/dev/cel/common/internal/AdaptingTypes.java +++ b/common/src/main/java/dev/cel/common/internal/AdaptingTypes.java @@ -23,7 +23,7 @@ import java.util.ListIterator; import java.util.Map; import java.util.Set; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * Collection of types which support bidirectional adaptation between CEL and Java native value diff --git a/common/src/main/java/dev/cel/common/internal/Errors.java b/common/src/main/java/dev/cel/common/internal/Errors.java index 3ee3ad19c..b2860355f 100644 --- a/common/src/main/java/dev/cel/common/internal/Errors.java +++ b/common/src/main/java/dev/cel/common/internal/Errors.java @@ -28,7 +28,7 @@ import java.util.Deque; import java.util.List; import java.util.Optional; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * An object which manages error reporting. Enriches error messages by source context pointing to diff --git a/common/src/main/java/dev/cel/common/internal/ProtoAdapter.java b/common/src/main/java/dev/cel/common/internal/ProtoAdapter.java index b63825dc4..29fece49c 100644 --- a/common/src/main/java/dev/cel/common/internal/ProtoAdapter.java +++ b/common/src/main/java/dev/cel/common/internal/ProtoAdapter.java @@ -59,7 +59,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * The {@code ProtoAdapter} utilities handle conversion between native Java objects which represent diff --git a/common/src/main/java/dev/cel/common/values/ListValue.java b/common/src/main/java/dev/cel/common/values/ListValue.java index 9cf6aa806..814884d34 100644 --- a/common/src/main/java/dev/cel/common/values/ListValue.java +++ b/common/src/main/java/dev/cel/common/values/ListValue.java @@ -24,7 +24,7 @@ import java.util.Comparator; import java.util.List; import java.util.function.UnaryOperator; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * ListValue is an abstract representation of a generic list containing zero or more {@link diff --git a/common/src/main/java/dev/cel/common/values/MapValue.java b/common/src/main/java/dev/cel/common/values/MapValue.java index ff1bb3f56..0f79b0464 100644 --- a/common/src/main/java/dev/cel/common/values/MapValue.java +++ b/common/src/main/java/dev/cel/common/values/MapValue.java @@ -26,7 +26,7 @@ import java.util.Optional; import java.util.function.BiFunction; import java.util.function.Function; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * MapValue is an abstract representation of a generic map containing {@link CelValue} as keys and diff --git a/common/src/main/java/dev/cel/common/values/OptionalValue.java b/common/src/main/java/dev/cel/common/values/OptionalValue.java index 45270ebb2..3866ef72b 100644 --- a/common/src/main/java/dev/cel/common/values/OptionalValue.java +++ b/common/src/main/java/dev/cel/common/values/OptionalValue.java @@ -21,7 +21,7 @@ import dev.cel.common.types.SimpleType; import java.util.NoSuchElementException; import java.util.Optional; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * First-class support for CEL optionals. Supports similar semantics to java.util.Optional. Also diff --git a/runtime/src/main/java/dev/cel/runtime/Activation.java b/runtime/src/main/java/dev/cel/runtime/Activation.java index c51e77e0c..1894c16ca 100644 --- a/runtime/src/main/java/dev/cel/runtime/Activation.java +++ b/runtime/src/main/java/dev/cel/runtime/Activation.java @@ -31,7 +31,7 @@ import java.util.HashMap; import java.util.Map; import java.util.Optional; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * An object which allows to bind names to values. diff --git a/runtime/src/main/java/dev/cel/runtime/CelRuntimeLegacyImpl.java b/runtime/src/main/java/dev/cel/runtime/CelRuntimeLegacyImpl.java index 9ef60304f..f5f96e5ca 100644 --- a/runtime/src/main/java/dev/cel/runtime/CelRuntimeLegacyImpl.java +++ b/runtime/src/main/java/dev/cel/runtime/CelRuntimeLegacyImpl.java @@ -48,7 +48,7 @@ import java.util.Map; import java.util.Optional; import java.util.function.Function; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * {@code CelRuntime} implementation based on the legacy CEL-Java stack. diff --git a/runtime/src/main/java/dev/cel/runtime/DescriptorMessageProvider.java b/runtime/src/main/java/dev/cel/runtime/DescriptorMessageProvider.java index 5c0931a00..f4fbbfe27 100644 --- a/runtime/src/main/java/dev/cel/runtime/DescriptorMessageProvider.java +++ b/runtime/src/main/java/dev/cel/runtime/DescriptorMessageProvider.java @@ -35,7 +35,7 @@ import dev.cel.common.types.CelTypes; import java.util.Map; import java.util.Optional; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * An implementation of {@link RuntimeTypeProvider} which relies on proto descriptors. diff --git a/runtime/src/main/java/dev/cel/runtime/DynamicMessageFactory.java b/runtime/src/main/java/dev/cel/runtime/DynamicMessageFactory.java index 60bfb6cd8..41f78b078 100644 --- a/runtime/src/main/java/dev/cel/runtime/DynamicMessageFactory.java +++ b/runtime/src/main/java/dev/cel/runtime/DynamicMessageFactory.java @@ -26,7 +26,7 @@ import dev.cel.common.internal.DefaultMessageFactory; import dev.cel.common.internal.ProtoMessageFactory; import java.util.Collection; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * The {@code DynamicMessageFactory} creates {@link DynamicMessage} instances by protobuf name. diff --git a/runtime/src/main/java/dev/cel/runtime/GlobalResolver.java b/runtime/src/main/java/dev/cel/runtime/GlobalResolver.java index 2cc1a7d31..a088ae2c1 100644 --- a/runtime/src/main/java/dev/cel/runtime/GlobalResolver.java +++ b/runtime/src/main/java/dev/cel/runtime/GlobalResolver.java @@ -15,7 +15,7 @@ package dev.cel.runtime; import dev.cel.common.annotations.Internal; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * An interface describing an object that can perform a lookup on a given name, returning the value diff --git a/runtime/src/main/java/dev/cel/runtime/InterpreterException.java b/runtime/src/main/java/dev/cel/runtime/InterpreterException.java index 98595ee88..e3e09aa24 100644 --- a/runtime/src/main/java/dev/cel/runtime/InterpreterException.java +++ b/runtime/src/main/java/dev/cel/runtime/InterpreterException.java @@ -20,7 +20,7 @@ import dev.cel.common.CelRuntimeException; import dev.cel.common.annotations.Internal; import dev.cel.common.internal.SafeStringFormatter; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * An exception produced during interpretation of expressions. diff --git a/runtime/src/main/java/dev/cel/runtime/InterpreterUtil.java b/runtime/src/main/java/dev/cel/runtime/InterpreterUtil.java index 05a764632..c2dac9cd0 100644 --- a/runtime/src/main/java/dev/cel/runtime/InterpreterUtil.java +++ b/runtime/src/main/java/dev/cel/runtime/InterpreterUtil.java @@ -23,7 +23,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Set; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * Util class for CEL interpreter. diff --git a/runtime/src/main/java/dev/cel/runtime/MessageFactory.java b/runtime/src/main/java/dev/cel/runtime/MessageFactory.java index e56825b4a..a7ec8de23 100644 --- a/runtime/src/main/java/dev/cel/runtime/MessageFactory.java +++ b/runtime/src/main/java/dev/cel/runtime/MessageFactory.java @@ -20,7 +20,7 @@ import dev.cel.common.internal.DefaultMessageFactory; import dev.cel.common.internal.ProtoMessageFactory; import java.util.Optional; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * The {@code MessageFactory} provides a method to create a protobuf builder objects by name. diff --git a/runtime/src/main/java/dev/cel/runtime/RuntimeTypeProviderLegacyImpl.java b/runtime/src/main/java/dev/cel/runtime/RuntimeTypeProviderLegacyImpl.java index 96dc40d0c..dcb2b4ec3 100644 --- a/runtime/src/main/java/dev/cel/runtime/RuntimeTypeProviderLegacyImpl.java +++ b/runtime/src/main/java/dev/cel/runtime/RuntimeTypeProviderLegacyImpl.java @@ -34,7 +34,7 @@ import dev.cel.common.values.StringValue; import java.util.Map; import java.util.NoSuchElementException; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** Bridge between the old RuntimeTypeProvider and CelValueProvider APIs. */ @Internal diff --git a/runtime/src/main/java/dev/cel/runtime/StandardTypeResolver.java b/runtime/src/main/java/dev/cel/runtime/StandardTypeResolver.java index 3a46ebea5..c560827cc 100644 --- a/runtime/src/main/java/dev/cel/runtime/StandardTypeResolver.java +++ b/runtime/src/main/java/dev/cel/runtime/StandardTypeResolver.java @@ -35,7 +35,7 @@ import java.util.Collection; import java.util.Map; import java.util.Optional; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * The {@code StandardTypeResolver} implements the {@link TypeResolver} and resolves types supported diff --git a/runtime/src/main/java/dev/cel/runtime/TypeResolver.java b/runtime/src/main/java/dev/cel/runtime/TypeResolver.java index 271c65bba..4bfa9ae7d 100644 --- a/runtime/src/main/java/dev/cel/runtime/TypeResolver.java +++ b/runtime/src/main/java/dev/cel/runtime/TypeResolver.java @@ -19,7 +19,7 @@ import com.google.errorprone.annotations.Immutable; import dev.cel.common.annotations.Internal; import dev.cel.common.types.CelType; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; /** * The {@code TypeResolver} determines the CEL type of Java-native values and assists with adapting diff --git a/runtime/src/test/java/dev/cel/runtime/RuntimeEqualityTest.java b/runtime/src/test/java/dev/cel/runtime/RuntimeEqualityTest.java index 250584f48..b75015213 100644 --- a/runtime/src/test/java/dev/cel/runtime/RuntimeEqualityTest.java +++ b/runtime/src/test/java/dev/cel/runtime/RuntimeEqualityTest.java @@ -52,7 +52,7 @@ import dev.cel.common.internal.DynamicProto; import java.util.Arrays; import java.util.List; -import org.jspecify.nullness.Nullable; +import org.jspecify.annotations.Nullable; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; From 6a1185be7c982668e871c2bbd842fb807d31e333 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Fri, 21 Jun 2024 12:59:13 -0700 Subject: [PATCH 04/33] Check for presence of optional indices to prevent autoboxing PiperOrigin-RevId: 645476381 --- .../src/main/java/dev/cel/runtime/DefaultInterpreter.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java index 7239edda3..b1b228803 100644 --- a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java +++ b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java @@ -711,7 +711,6 @@ private IntermediateResult evalBooleanNonstrict(ExecutionFrame frame, CelExpr ex private IntermediateResult evalList(ExecutionFrame frame, CelExpr unusedExpr, CelList listExpr) throws InterpreterException { - CallArgumentChecker argChecker = CallArgumentChecker.create(frame.getResolver()); List result = new ArrayList<>(listExpr.elements().size()); @@ -726,7 +725,11 @@ private IntermediateResult evalList(ExecutionFrame frame, CelExpr unusedExpr, Ce argChecker.checkArg(evaluatedElement); Object value = evaluatedElement.value(); - if (optionalIndicesSet.contains(i) && !isUnknownValue(value)) { + if (!optionalIndicesSet + .isEmpty() // Performance optimization to prevent autoboxing when there's no + // optionals. + && optionalIndicesSet.contains(i) + && !isUnknownValue(value)) { Optional optionalVal = (Optional) value; if (!optionalVal.isPresent()) { continue; From f2d263552bc0f57c7dddec468aeaae8229925974 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 24 Jun 2024 11:01:19 -0700 Subject: [PATCH 05/33] Remove PartialMessage and IncompleteData PiperOrigin-RevId: 646154419 --- .../src/main/java/dev/cel/runtime/BUILD.bazel | 5 - .../dev/cel/runtime/DefaultInterpreter.java | 18 +- .../java/dev/cel/runtime/IncompleteData.java | 26 - .../java/dev/cel/runtime/InterpreterUtil.java | 23 - .../java/dev/cel/runtime/PartialMessage.java | 181 ----- .../cel/runtime/PartialMessageOrBuilder.java | 46 -- .../cel/runtime/CelValueInterpreterTest.java | 15 - .../src/test/resources/unknownField.baseline | 518 +------------- .../test/resources/unknownResultSet.baseline | 657 +++--------------- .../dev/cel/testing/BaseInterpreterTest.java | 200 ++---- 10 files changed, 168 insertions(+), 1521 deletions(-) delete mode 100644 runtime/src/main/java/dev/cel/runtime/IncompleteData.java delete mode 100644 runtime/src/main/java/dev/cel/runtime/PartialMessage.java delete mode 100644 runtime/src/main/java/dev/cel/runtime/PartialMessageOrBuilder.java diff --git a/runtime/src/main/java/dev/cel/runtime/BUILD.bazel b/runtime/src/main/java/dev/cel/runtime/BUILD.bazel index 335c6e332..f51706987 100644 --- a/runtime/src/main/java/dev/cel/runtime/BUILD.bazel +++ b/runtime/src/main/java/dev/cel/runtime/BUILD.bazel @@ -8,7 +8,6 @@ package( BASE_SOURCES = [ "DefaultMetadata.java", - "IncompleteData.java", "InterpreterException.java", "MessageProvider.java", "Metadata.java", @@ -31,8 +30,6 @@ INTERPRETER_SOURCES = [ "Interpreter.java", "InterpreterUtil.java", "MessageFactory.java", - "PartialMessage.java", - "PartialMessageOrBuilder.java", "RuntimeTypeProvider.java", "RuntimeUnknownResolver.java", "UnknownTrackingInterpretable.java", @@ -98,7 +95,6 @@ java_library( "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", "@maven//:com_google_protobuf_protobuf_java", - "@maven//:com_google_protobuf_protobuf_java_util", "@maven//:org_jspecify_jspecify", ], ) @@ -247,7 +243,6 @@ java_library( ], deps = [ ":base", - "//common:error_codes", "//common/annotations", "@cel_spec//proto/cel/expr:expr_java_proto", "@maven//:com_google_errorprone_error_prone_annotations", diff --git a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java index b1b228803..b35c4dea6 100644 --- a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java +++ b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java @@ -177,10 +177,7 @@ public Object evalTrackingUnknowns( ExecutionFrame frame = new ExecutionFrame(listener, resolver, celOptions.comprehensionMaxIterations()); IntermediateResult internalResult = evalInternal(frame, ast.getExpr()); - Object result = internalResult.value(); - // TODO: remove support for IncompleteData. - return InterpreterUtil.completeDataOnly( - result, "Incomplete data cannot be returned as a result."); + return internalResult.value(); } private IntermediateResult evalInternal(ExecutionFrame frame, CelExpr expr) @@ -388,10 +385,6 @@ private IntermediateResult evalCall(ExecutionFrame frame, CelExpr expr, CelCall // Default evaluation is strict so errors will propagate (via thrown Java exception) before // unknowns. argResults[i] = evalInternal(frame, callArgs.get(i)); - // TODO: remove support for IncompleteData after migrating users to attribute - // tracking unknowns. - InterpreterUtil.completeDataOnly( - argResults[i].value(), "Incomplete data does not support function calls."); } Optional indexAttr = @@ -719,9 +712,6 @@ private IntermediateResult evalList(ExecutionFrame frame, CelExpr unusedExpr, Ce for (int i = 0; i < elements.size(); i++) { CelExpr element = elements.get(i); IntermediateResult evaluatedElement = evalInternal(frame, element); - // TODO: remove support for IncompleteData. - InterpreterUtil.completeDataOnly( - evaluatedElement.value(), "Incomplete data cannot be an elem of a list."); argChecker.checkArg(evaluatedElement); Object value = evaluatedElement.value(); @@ -756,9 +746,6 @@ private IntermediateResult evalMap(ExecutionFrame frame, CelMap mapExpr) argChecker.checkArg(keyResult); IntermediateResult valueResult = evalInternal(frame, entry.value()); - // TODO: remove support for IncompleteData. - InterpreterUtil.completeDataOnly( - valueResult.value(), "Incomplete data cannot be a value of a map."); argChecker.checkArg(valueResult); if (celOptions.errorOnDuplicateMapKeys() && result.containsKey(keyResult.value())) { @@ -801,9 +788,6 @@ private IntermediateResult evalStruct(ExecutionFrame frame, CelExpr expr, CelStr Map fields = new HashMap<>(); for (CelStruct.Entry entry : structExpr.entries()) { IntermediateResult fieldResult = evalInternal(frame, entry.value()); - // TODO: remove support for IncompleteData - InterpreterUtil.completeDataOnly( - fieldResult.value(), "Incomplete data cannot be a field of a message."); argChecker.checkArg(fieldResult); Object value = fieldResult.value(); diff --git a/runtime/src/main/java/dev/cel/runtime/IncompleteData.java b/runtime/src/main/java/dev/cel/runtime/IncompleteData.java deleted file mode 100644 index 4859f63c6..000000000 --- a/runtime/src/main/java/dev/cel/runtime/IncompleteData.java +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dev.cel.runtime; - -import dev.cel.common.annotations.Internal; - -/** - * Add an interface for interpreter to check if an object is an instance of {@link PartialMessage}. - * - *

Deprecated. New clients should use {@link CelAttribute} based unknowns. - */ -@Deprecated -@Internal -public interface IncompleteData {} diff --git a/runtime/src/main/java/dev/cel/runtime/InterpreterUtil.java b/runtime/src/main/java/dev/cel/runtime/InterpreterUtil.java index c2dac9cd0..7f2808c0e 100644 --- a/runtime/src/main/java/dev/cel/runtime/InterpreterUtil.java +++ b/runtime/src/main/java/dev/cel/runtime/InterpreterUtil.java @@ -16,8 +16,6 @@ import dev.cel.expr.ExprValue; import dev.cel.expr.UnknownSet; -import com.google.errorprone.annotations.CanIgnoreReturnValue; -import dev.cel.common.CelErrorCode; import dev.cel.common.annotations.Internal; import java.util.Arrays; import java.util.LinkedHashSet; @@ -62,27 +60,6 @@ public static boolean isUnknown(Object obj) { && ((ExprValue) obj).getKindCase() == ExprValue.KindCase.UNKNOWN; } - /** - * Throws an InterpreterException with {@code exceptionMessage} if the {@code obj} is an instance - * of {@link IncompleteData}. {@link IncompleteData} does not support some operators. - * - *

Returns the obj argument otherwise. - * - *

Deprecated. TODO: Can be removed once clients have stopped using - * IncompleteData. - */ - @CanIgnoreReturnValue - @Deprecated - public static Object completeDataOnly(Object obj, String exceptionMessage) - throws InterpreterException { - if (obj instanceof IncompleteData) { - throw new InterpreterException.Builder(exceptionMessage) - .setErrorCode(CelErrorCode.INVALID_ARGUMENT) - .build(); - } - return obj; - } - /** * Combine multiple ExprValue objects which has UnknownSet into one ExprValue * diff --git a/runtime/src/main/java/dev/cel/runtime/PartialMessage.java b/runtime/src/main/java/dev/cel/runtime/PartialMessage.java deleted file mode 100644 index 0d9ad5dca..000000000 --- a/runtime/src/main/java/dev/cel/runtime/PartialMessage.java +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dev.cel.runtime; - -import com.google.protobuf.Descriptors; -import com.google.protobuf.Descriptors.Descriptor; -import com.google.protobuf.Descriptors.FieldDescriptor; -import com.google.protobuf.Descriptors.OneofDescriptor; -import com.google.protobuf.FieldMask; -import com.google.protobuf.Message; -import com.google.protobuf.UnknownFieldSet; -import com.google.protobuf.util.FieldMaskUtil; -import dev.cel.common.annotations.Internal; -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -/** - * Wrap a Message to throw an error on access to certain fields or sub-fields, as described by a - * FieldMask. - * - *

Deprecated. New clients should use {@link CelAttribute} based unknowns. - */ -@Deprecated -@Internal -public class PartialMessage implements PartialMessageOrBuilder, IncompleteData { - - private final Message message; - private final FieldMask fieldMask; - - @Override - public Message getDefaultInstanceForType() { - return message.getDefaultInstanceForType(); - } - - @Override - public boolean isInitialized() { - return message.isInitialized(); - } - - @Override - public List findInitializationErrors() { - return message.findInitializationErrors(); - } - - @Override - public String getInitializationErrorString() { - return message.getInitializationErrorString(); - } - - @Override - public Descriptor getDescriptorForType() { - return message.getDescriptorForType(); - } - - @Override - public Map getAllFields() { - return message.getAllFields(); - } - - @Override - public boolean hasOneof(OneofDescriptor oneof) { - return message.hasOneof(oneof); - } - - @Override - public FieldDescriptor getOneofFieldDescriptor(OneofDescriptor oneof) { - return message.getOneofFieldDescriptor(oneof); - } - - @Override - public boolean hasField(FieldDescriptor field) { - return message.hasField(field); - } - - /** Create relative field masks to the field specified by the name. */ - private FieldMask subpathMask(String name) { - FieldMask.Builder builder = FieldMask.newBuilder(); - for (String p : fieldMask.getPathsList()) { - if (!p.startsWith(name)) { - continue; - } - String tmp = p.substring(name.length() + 1); - if (tmp.length() > 0) { - builder.addPaths(tmp); - } - } - return builder.build(); - } - - @Override - public Object getField(Descriptors.FieldDescriptor field) { - String path = field.getName(); - - if (fieldMask.getPathsList().contains(path)) { - return InterpreterUtil.createUnknownExprValue(new ArrayList()); - } - - Object obj = message.getField(field); - FieldMask subFieldMask = subpathMask(path); - if (obj instanceof Message && !subFieldMask.getPathsList().isEmpty()) { - // Partial message means at least one of its field has been marked as unknown - return new PartialMessage((Message) obj, subFieldMask); - } else { - return obj; - } - } - - @Override - public int getRepeatedFieldCount(FieldDescriptor field) { - return message.getRepeatedFieldCount(field); - } - - @Override - public Object getRepeatedField(FieldDescriptor field, int index) { - return message.getRepeatedField(field, index); - } - - @Override - public UnknownFieldSet getUnknownFields() { - return message.getUnknownFields(); - } - - public PartialMessage(Message m) { - this.message = m; - this.fieldMask = FieldMask.getDefaultInstance(); - } - - public PartialMessage(Message m, FieldMask mask) { - this.message = m; - this.fieldMask = mask; - - if (m == null) { - throw new NullPointerException("The message in PartialMessage is null."); - } - if (!FieldMaskUtil.isValid(m.getDescriptorForType(), fieldMask)) { - throw new RuntimeException( - new InterpreterException.Builder("Invalid field mask for message:" + message).build()); - } - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append(this.getClass()); - sb.append("{\nmessage: {\n"); - sb.append(this.message); - sb.append("},\nfieldMask: {\n"); - for (Iterator it = fieldMask.getPathsList().iterator(); it.hasNext(); ) { - sb.append(" paths: ").append(it.next()); - if (it.hasNext()) { - sb.append(",\n"); - } - } - sb.append("\n}\n"); - return sb.toString(); - } - - @Override - public Message getMessage() { - return message; - } - - @Override - public FieldMask getFieldMask() { - return fieldMask; - } -} diff --git a/runtime/src/main/java/dev/cel/runtime/PartialMessageOrBuilder.java b/runtime/src/main/java/dev/cel/runtime/PartialMessageOrBuilder.java deleted file mode 100644 index 9009ea37e..000000000 --- a/runtime/src/main/java/dev/cel/runtime/PartialMessageOrBuilder.java +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright 2022 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package dev.cel.runtime; - -import com.google.protobuf.Descriptors.FieldDescriptor; -import com.google.protobuf.FieldMask; -import com.google.protobuf.Message; -import com.google.protobuf.MessageOrBuilder; -import dev.cel.common.annotations.Internal; - -/** - * Wrap Message to support Unknown value. - * - *

Deprecated. New clients should use {@link CelAttribute} based unknowns. - */ -@Deprecated -@Internal -public interface PartialMessageOrBuilder extends MessageOrBuilder { - /** Return original message. */ - public Message getMessage(); - - /* - * Return field mask. - */ - public FieldMask getFieldMask(); - - /** - * This method is similar to {@link MessageOrBuilder#getField(FieldDescriptor)}, with the - * following differences: This method may throw an InterpreterException wrapped with a - * RuntimeException, if the field path is set in the Field mask. - */ - @Override - public Object getField(FieldDescriptor field); -} diff --git a/runtime/src/test/java/dev/cel/runtime/CelValueInterpreterTest.java b/runtime/src/test/java/dev/cel/runtime/CelValueInterpreterTest.java index 4c657a0c8..2ef7a250f 100644 --- a/runtime/src/test/java/dev/cel/runtime/CelValueInterpreterTest.java +++ b/runtime/src/test/java/dev/cel/runtime/CelValueInterpreterTest.java @@ -22,7 +22,6 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; -import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.junit.runners.Parameterized.Parameters; @@ -44,20 +43,6 @@ public CelValueInterpreterTest(boolean declareWithCelType, Eval eval) { super(declareWithCelType, eval); } - /** Test relies on PartialMessage, which is deprecated and not supported for CelValue. */ - @Override - @Test - public void unknownField() { - skipBaselineVerification(); - } - - /** Test relies on PartialMessage, which is deprecated and not supported for CelValue. */ - @Override - @Test - public void unknownResultSet() { - skipBaselineVerification(); - } - @Parameters public static List testData() { return new ArrayList<>( diff --git a/runtime/src/test/resources/unknownField.baseline b/runtime/src/test/resources/unknownField.baseline index 2831bfd9b..92f0a7385 100644 --- a/runtime/src/test/resources/unknownField.baseline +++ b/runtime/src/test/resources/unknownField.baseline @@ -3,30 +3,9 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } @@ -35,30 +14,9 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } @@ -67,30 +25,9 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } @@ -99,465 +36,40 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} -result: 15 - -Source: x.single_duration.getMilliseconds() -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} +bindings: {} +result: unknown { + exprs: 1 } -error: evaluation error: Incomplete data does not support function calls. -error_code: INVALID_ARGUMENT -Source: x.single_duration + x.single_duration -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} -error: evaluation error: Incomplete data does not support function calls. -error_code: INVALID_ARGUMENT Source: x.single_nested_message.bb declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} +bindings: {} result: unknown { - exprs: 3 + exprs: 1 } -Source: x.single_nested_message -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} -error: evaluation error: Incomplete data cannot be returned as a result. -error_code: INVALID_ARGUMENT - -Source: TestAllTypes{single_nested_message: x.single_nested_message} -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} -error: evaluation error: Incomplete data cannot be a field of a message. -error_code: INVALID_ARGUMENT - Source: {1: x.single_int32} declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} -result: unknown { - exprs: 5 -} - - -Source: {1: x.single_int64} -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} -result: {1=0} - -Source: {1: x.single_nested_message} -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} -error: evaluation error: Incomplete data cannot be a value of a map. -error_code: INVALID_ARGUMENT - -Source: [1, x.single_int32] -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} +bindings: {} result: unknown { exprs: 4 } -Source: [x.single_nested_message] -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: map_int32_int64, - paths: single_int32, - paths: single_nested_message.bb, - paths: repeated_nested_message, - paths: single_duration.seconds -} -} -error: evaluation error: Incomplete data cannot be an elem of a list. -error_code: INVALID_ARGUMENT - -Source: x.single_nested_message.bb -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_nested_message -} -} -result: unknown { - exprs: 2 -} - - -Source: (x.single_nested_message.bb == 42) || true -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_nested_message -} -} -result: true - -Source: (x.single_nested_message.bb == 42) || false -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_nested_message -} -} -result: unknown { - exprs: 2 -} - - -Source: (x.single_nested_message.bb == 42) && true +Source: [1, x.single_int32] declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_nested_message -} -} +bindings: {} result: unknown { - exprs: 2 -} - - -Source: (x.single_nested_message.bb == 42) && false -declare x { - value dev.cel.testing.testdata.proto3.TestAllTypes -} -=====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_nested_message { -} -repeated_nested_message { - bb: 14 -} -single_duration { - seconds: 15 -} -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_nested_message -} + exprs: 3 } -result: false diff --git a/runtime/src/test/resources/unknownResultSet.baseline b/runtime/src/test/resources/unknownResultSet.baseline index aa9d032ff..eb7532b8b 100644 --- a/runtime/src/test/resources/unknownResultSet.baseline +++ b/runtime/src/test/resources/unknownResultSet.baseline @@ -1,44 +1,20 @@ -Source: x.single_int32 == 1 && x.single_string == "test" +Source: x.single_int32 == 1 && true declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } -Source: x.single_int32 == 1 && x.single_string != "test" +Source: x.single_int32 == 1 && false declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: false Source: x.single_int32 == 1 && x.single_int64 == 1 @@ -46,22 +22,10 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 - exprs: 7 + exprs: 1 + exprs: 6 } @@ -70,65 +34,29 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } -Source: x.single_string == "test" && x.single_int32 == 1 +Source: true && x.single_int32 == 1 declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 7 + exprs: 3 } -Source: x.single_string != "test" && x.single_int32 == 1 +Source: false && x.single_int32 == 1 declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: false Source: x.single_timestamp <= timestamp("bad timestamp string") && x.single_int32 == 1 @@ -136,21 +64,9 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 8 + exprs: 7 } @@ -159,19 +75,7 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} error: evaluation error: Failed to parse timestamp: invalid timestamp "bad timestamp string" error_code: BAD_FORMAT @@ -180,41 +84,22 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} +bindings: {} +result: unknown { + exprs: 1 + exprs: 6 } -result: true + Source: x.single_int32 == 1 || x.single_string != "test" declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 + exprs: 6 } @@ -223,22 +108,10 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 - exprs: 7 + exprs: 1 + exprs: 6 } @@ -247,64 +120,28 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } -Source: x.single_string == "test" || x.single_int32 == 1 +Source: true || x.single_int32 == 1 declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: true -Source: x.single_string != "test" || x.single_int32 == 1 +Source: false || x.single_int32 == 1 declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 7 + exprs: 3 } @@ -313,21 +150,9 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 8 + exprs: 7 } @@ -336,19 +161,7 @@ declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} error: evaluation error: Failed to parse timestamp: invalid timestamp "bad timestamp string" error_code: BAD_FORMAT @@ -360,21 +173,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } @@ -386,21 +187,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 4 + exprs: 3 } @@ -412,22 +201,10 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 - exprs: 5 + exprs: 1 + exprs: 4 } @@ -439,18 +216,10 @@ declare f { function f int.(int) -> bool } =====> -bindings: {y=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" +bindings: {y=single_string: "test" single_timestamp { seconds: 15 } -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} } result: unknown { exprs: 1 @@ -481,21 +250,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } @@ -507,19 +264,7 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: true Source: [0, 2, 4].exists(z, z == x.single_int32) @@ -530,21 +275,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 10 + exprs: 9 } @@ -556,21 +289,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 27 + exprs: 26 } @@ -582,21 +303,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 13 + exprs: 12 } @@ -608,21 +317,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 27 + exprs: 26 } @@ -634,22 +331,10 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 18 - exprs: 27 + exprs: 17 + exprs: 26 } @@ -661,25 +346,13 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } -Source: x.single_string == "test" ? x.single_int32 : 2 +Source: true ? x.single_int32 : 2 declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } @@ -687,25 +360,13 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 7 + exprs: 3 } -Source: x.single_string == "test" ? 1 : x.single_int32 +Source: true ? 1 : x.single_int32 declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } @@ -713,22 +374,10 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: 1 -Source: x.single_string != "test" ? x.single_int32 : 2 +Source: false ? x.single_int32 : 2 declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } @@ -736,22 +385,10 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: 2 -Source: x.single_string != "test" ? 1 : x.single_int32 +Source: false ? 1 : x.single_int32 declare x { value dev.cel.testing.testdata.proto3.TestAllTypes } @@ -759,21 +396,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 8 + exprs: 4 } @@ -785,21 +410,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 2 + exprs: 1 } @@ -811,21 +424,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 4 + exprs: 3 } @@ -837,21 +438,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 5 + exprs: 4 } @@ -863,22 +452,10 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 5 - exprs: 8 + exprs: 4 + exprs: 7 } @@ -890,21 +467,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 4 + exprs: 3 } @@ -916,22 +481,10 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 4 - exprs: 6 + exprs: 3 + exprs: 5 } @@ -943,21 +496,9 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 4 + exprs: 3 } @@ -969,22 +510,8 @@ declare f { function f int.(int) -> bool } =====> -bindings: {x=class dev.cel.runtime.PartialMessage{ -message: { -single_string: "test" -single_timestamp { - seconds: 15 -} -}, -fieldMask: { - paths: single_int32, - paths: single_int64, - paths: map_int32_int64 -} -} +bindings: {} result: unknown { - exprs: 4 - exprs: 7 + exprs: 3 + exprs: 6 } - - diff --git a/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java b/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java index 43fadd81c..bc9afa55e 100644 --- a/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java +++ b/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java @@ -40,7 +40,6 @@ import com.google.protobuf.DoubleValue; import com.google.protobuf.Duration; import com.google.protobuf.DynamicMessage; -import com.google.protobuf.FieldMask; import com.google.protobuf.FloatValue; import com.google.protobuf.Int32Value; import com.google.protobuf.Int64Value; @@ -63,7 +62,6 @@ import dev.cel.common.types.CelTypes; import dev.cel.runtime.Activation; import dev.cel.runtime.InterpreterException; -import dev.cel.runtime.PartialMessage; import dev.cel.testing.testdata.proto3.StandaloneGlobalEnum; import dev.cel.testing.testdata.proto3.TestAllTypesProto.TestAllTypes; import dev.cel.testing.testdata.proto3.TestAllTypesProto.TestAllTypes.NestedEnum; @@ -1012,189 +1010,111 @@ public void timestampFunctions() throws Exception { public void unknownField() throws Exception { container = TestAllTypes.getDescriptor().getFile().getPackage(); declareVariable("x", CelTypes.createMessage(TestAllTypes.getDescriptor().getFullName())); - TestAllTypes val = - TestAllTypes.newBuilder() - .setSingleTimestamp(Timestamps.fromSeconds(15)) - .setSingleDuration(Durations.fromSeconds(15)) - .setSingleNestedMessage(NestedMessage.getDefaultInstance()) - .addRepeatedNestedMessage(0, NestedMessage.newBuilder().setBb(14).build()) - .build(); - - PartialMessage wm = - new PartialMessage( - val, - FieldMask.newBuilder() - .addPaths("map_int32_int64") - .addPaths("single_int32") - .addPaths("single_nested_message.bb") - .addPaths("repeated_nested_message") - .addPaths("single_duration.seconds") - .build()); // Unknown field is accessed. source = "x.single_int32"; - runTest(Activation.of("x", wm)); + runTest(Activation.EMPTY); source = "x.map_int32_int64[22]"; - runTest(Activation.of("x", wm)); + runTest(Activation.EMPTY); source = "x.repeated_nested_message[1]"; - runTest(Activation.of("x", wm)); + runTest(Activation.EMPTY); - // Function call for a known field. + // Function call for an unknown field. source = "x.single_timestamp.getSeconds()"; - runTest(Activation.of("x", wm)); - - // PartialMessage does not support function call - source = "x.single_duration.getMilliseconds()"; - runTest(Activation.of("x", wm)); - - // PartialMessage does not support operators. - source = "x.single_duration + x.single_duration"; - runTest(Activation.of("x", wm)); + runTest(Activation.EMPTY); // Unknown field in a nested message source = "x.single_nested_message.bb"; - runTest(Activation.of("x", wm)); - - // PartialMessage cannot be a final expr result. - source = "x.single_nested_message"; - runTest(Activation.of("x", wm)); - - // PartialMessage cannot be a field of another message. - source = "TestAllTypes{single_nested_message: x.single_nested_message}"; - runTest(Activation.of("x", wm)); + runTest(Activation.EMPTY); - // Unknown field cannot be a value of a map now. + // Unknown field access in a map. source = "{1: x.single_int32}"; - runTest(Activation.of("x", wm)); - - // Access a known field as a val of a map. - source = "{1: x.single_int64}"; - runTest(Activation.of("x", wm)); - - // PartialMessage cannot be a value of a map. - source = "{1: x.single_nested_message}"; - runTest(Activation.of("x", wm)); + runTest(Activation.EMPTY); - // Unknown field cannot be a value of a list now. + // Unknown field access in a list. source = "[1, x.single_int32]"; - runTest(Activation.of("x", wm)); - - // PartialMessage cannot be an elem in a list. - source = "[x.single_nested_message]"; - runTest(Activation.of("x", wm)); - - // Access a field in a nested message masked as unknown. - wm = new PartialMessage(val, FieldMask.newBuilder().addPaths("single_nested_message").build()); - - // Access unknown field. - source = "x.single_nested_message.bb"; - runTest(Activation.of("x", wm)); - - // Error or true should be true. - source = "(x.single_nested_message.bb == 42) || true"; - runTest(Activation.of("x", wm)); - - // Error or false should be error. - source = "(x.single_nested_message.bb == 42) || false"; - runTest(Activation.of("x", wm)); - - // Error and true should be error. - source = "(x.single_nested_message.bb == 42) && true"; - runTest(Activation.of("x", wm)); - - // Error and false should be false. - source = "(x.single_nested_message.bb == 42) && false"; - runTest(Activation.of("x", wm)); + runTest(Activation.EMPTY); } @Test public void unknownResultSet() throws Exception { container = TestAllTypes.getDescriptor().getFile().getPackage(); declareVariable("x", CelTypes.createMessage(TestAllTypes.getDescriptor().getFullName())); - TestAllTypes val = + TestAllTypes message = TestAllTypes.newBuilder() .setSingleString("test") .setSingleTimestamp(Timestamp.newBuilder().setSeconds(15)) .build(); - PartialMessage message = - new PartialMessage( - val, - FieldMask.newBuilder() - .addPaths("single_int32") - .addPaths("single_int64") - .addPaths("map_int32_int64") - .build()); - // unknown && true ==> unknown - source = "x.single_int32 == 1 && x.single_string == \"test\""; - runTest(Activation.of("x", message)); + source = "x.single_int32 == 1 && true"; + runTest(Activation.EMPTY); // unknown && false ==> false - source = "x.single_int32 == 1 && x.single_string != \"test\""; - runTest(Activation.of("x", message)); + source = "x.single_int32 == 1 && false"; + runTest(Activation.EMPTY); // unknown && Unknown ==> UnknownSet source = "x.single_int32 == 1 && x.single_int64 == 1"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // unknown && error ==> unknown source = "x.single_int32 == 1 && x.single_timestamp <= timestamp(\"bad timestamp string\")"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // true && unknown ==> unknown - source = "x.single_string == \"test\" && x.single_int32 == 1"; - runTest(Activation.of("x", message)); + source = "true && x.single_int32 == 1"; + runTest(Activation.EMPTY); // false && unknown ==> false - source = "x.single_string != \"test\" && x.single_int32 == 1"; - runTest(Activation.of("x", message)); + source = "false && x.single_int32 == 1"; + runTest(Activation.EMPTY); // error && unknown ==> unknown source = "x.single_timestamp <= timestamp(\"bad timestamp string\") && x.single_int32 == 1"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // error && error ==> error source = "x.single_timestamp <= timestamp(\"bad timestamp string\") " + "&& x.single_timestamp > timestamp(\"another bad timestamp string\")"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // unknown || true ==> true source = "x.single_int32 == 1 || x.single_string == \"test\""; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // unknown || false ==> unknown source = "x.single_int32 == 1 || x.single_string != \"test\""; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // unknown || unknown ==> UnknownSet source = "x.single_int32 == 1 || x.single_int64 == 1"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // unknown || error ==> unknown source = "x.single_int32 == 1 || x.single_timestamp <= timestamp(\"bad timestamp string\")"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // true || unknown ==> true - source = "x.single_string == \"test\" || x.single_int32 == 1"; - runTest(Activation.of("x", message)); + source = "true || x.single_int32 == 1"; + runTest(Activation.EMPTY); // false || unknown ==> unknown - source = "x.single_string != \"test\" || x.single_int32 == 1"; - runTest(Activation.of("x", message)); + source = "false || x.single_int32 == 1"; + runTest(Activation.EMPTY); // error || unknown ==> unknown source = "x.single_timestamp <= timestamp(\"bad timestamp string\") || x.single_int32 == 1"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // error || error ==> error source = "x.single_timestamp <= timestamp(\"bad timestamp string\") " + "|| x.single_timestamp > timestamp(\"another bad timestamp string\")"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // dispatch test declareFunction( @@ -1203,15 +1123,15 @@ public void unknownResultSet() throws Exception { // dispatch: unknown.f(1) ==> unknown source = "x.single_int32.f(1)"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // dispatch: 1.f(unknown) ==> unknown source = "1.f(x.single_int32)"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // dispatch: unknown.f(unknown) ==> unknownSet source = "x.single_int64.f(x.single_int32)"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // ident is null(x is unbound) ==> unknown source = "x"; @@ -1226,91 +1146,91 @@ public void unknownResultSet() throws Exception { // comprehension test // iteRange is unknown => unknown source = "x.map_int32_int64.map(x, x > 0, x + 1)"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // exists, loop condition encounters unknown => skip unknown and check other element source = "[0, 2, 4].exists(z, z == 2 || z == x.single_int32)"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // exists, loop condition encounters unknown => skip unknown and check other element, no dupe id // in result source = "[0, 2, 4].exists(z, z == x.single_int32)"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // exists_one, loop condition encounters unknown => collect all unknowns source = "[0, 2, 4].exists_one(z, z == 0 || (z == 2 && z == x.single_int32) " + "|| (z == 4 && z == x.single_int64))"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // all, loop condition encounters unknown => skip unknown and check other element source = "[0, 2].all(z, z == 2 || z == x.single_int32)"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // filter, loop condition encounters unknown => skip unknown and check other element source = "[0, 2, 4].filter(z, z == 0 || (z == 2 && z == x.single_int32) " + "|| (z == 4 && z == x.single_int64))"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // map, loop condition encounters unknown => skip unknown and check other element source = "[0, 2, 4].map(z, z == 0 || (z == 2 && z == x.single_int32) " + "|| (z == 4 && z == x.single_int64))"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // conditional test // unknown ? 1 : 2 ==> unknown source = "x.single_int32 == 1 ? 1 : 2"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // true ? unknown : 2 ==> unknown - source = "x.single_string == \"test\" ? x.single_int32 : 2"; - runTest(Activation.of("x", message)); + source = "true ? x.single_int32 : 2"; + runTest(Activation.EMPTY); // true ? 1 : unknown ==> 1 - source = "x.single_string == \"test\" ? 1 : x.single_int32"; - runTest(Activation.of("x", message)); + source = "true ? 1 : x.single_int32"; + runTest(Activation.EMPTY); // false ? unknown : 2 ==> 2 - source = "x.single_string != \"test\" ? x.single_int32 : 2"; - runTest(Activation.of("x", message)); + source = "false ? x.single_int32 : 2"; + runTest(Activation.EMPTY); // false ? 1 : unknown ==> unknown - source = "x.single_string != \"test\" ? 1 : x.single_int32"; - runTest(Activation.of("x", message)); + source = "false ? 1 : x.single_int32"; + runTest(Activation.EMPTY); // unknown condition ? unknown : unknown ==> unknown condition source = "x.single_int64 == 1 ? x.single_int32 : x.single_int32"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // map with unknown key => unknown source = "{x.single_int32: 2, 3: 4}"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // map with unknown value => unknown source = "{1: x.single_int32, 3: 4}"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // map with unknown key and value => unknownSet source = "{1: x.single_int32, x.single_int64: 4}"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // list with unknown => unknown source = "[1, x.single_int32, 3, 4]"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // list with multiple unknowns => unknownSet source = "[1, x.single_int32, x.single_int64, 4]"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // message with unknown => unknown source = "TestAllTypes{single_int32: x.single_int32}.single_int32 == 2"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); // message with multiple unknowns => unknownSet source = "TestAllTypes{single_int32: x.single_int32, single_int64: x.single_int64}"; - runTest(Activation.of("x", message)); + runTest(Activation.EMPTY); } @Test From 0e2fd64e430ffdc00b4fb873bca6228e936d3215 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Tue, 25 Jun 2024 13:04:20 -0700 Subject: [PATCH 06/33] Remove celVarToDecl method PiperOrigin-RevId: 646579113 --- common/src/main/java/dev/cel/common/CelVarDecl.java | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/common/src/main/java/dev/cel/common/CelVarDecl.java b/common/src/main/java/dev/cel/common/CelVarDecl.java index 9c3930c3a..0543ed225 100644 --- a/common/src/main/java/dev/cel/common/CelVarDecl.java +++ b/common/src/main/java/dev/cel/common/CelVarDecl.java @@ -14,13 +14,9 @@ package dev.cel.common; -import dev.cel.expr.Decl; -import dev.cel.expr.Decl.IdentDecl; import com.google.auto.value.AutoValue; -import com.google.common.annotations.VisibleForTesting; import com.google.errorprone.annotations.CheckReturnValue; import dev.cel.common.types.CelType; -import dev.cel.common.types.CelTypes; /** Abstract representation of a CEL variable declaration. */ @AutoValue @@ -37,13 +33,4 @@ public abstract class CelVarDecl { public static CelVarDecl newVarDeclaration(String name, CelType type) { return new AutoValue_CelVarDecl(name, type); } - - /** Converts a {@link CelVarDecl} to a protobuf equivalent form {@code Decl} */ - @VisibleForTesting - public static Decl celVarToDecl(CelVarDecl varDecl) { - return Decl.newBuilder() - .setName(varDecl.name()) - .setIdent(IdentDecl.newBuilder().setType(CelTypes.celTypeToType(varDecl.type()))) - .build(); - } } From db83a4704459edbbe6fc0139742f0f100c4a702c Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Fri, 28 Jun 2024 13:37:37 -0700 Subject: [PATCH 07/33] Adapt the function dispatch result to allow for automatic Any unpacking PiperOrigin-RevId: 647784646 --- .../dev/cel/runtime/DefaultInterpreter.java | 9 +-- .../src/test/resources/packUnpackAny.baseline | 59 +++++++++++++++++++ .../dev/cel/testing/BaseInterpreterTest.java | 5 ++ 3 files changed, 69 insertions(+), 4 deletions(-) diff --git a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java index b35c4dea6..80622c62c 100644 --- a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java +++ b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java @@ -411,10 +411,11 @@ private IntermediateResult evalCall(ExecutionFrame frame, CelExpr expr, CelCall Object[] argArray = Arrays.stream(argResults).map(IntermediateResult::value).toArray(); - return IntermediateResult.create( - attr, + Object dispatchResult = dispatcher.dispatch( - metadata, expr.id(), callExpr.function(), reference.overloadIds(), argArray)); + metadata, expr.id(), callExpr.function(), reference.overloadIds(), argArray); + dispatchResult = typeProvider.adapt(dispatchResult); + return IntermediateResult.create(attr, dispatchResult); } private Optional maybeContainerIndexAttribute( @@ -717,7 +718,7 @@ private IntermediateResult evalList(ExecutionFrame frame, CelExpr unusedExpr, Ce Object value = evaluatedElement.value(); if (!optionalIndicesSet .isEmpty() // Performance optimization to prevent autoboxing when there's no - // optionals. + // optionals. && optionalIndicesSet.contains(i) && !isUnknownValue(value)) { Optional optionalVal = (Optional) value; diff --git a/runtime/src/test/resources/packUnpackAny.baseline b/runtime/src/test/resources/packUnpackAny.baseline index 44466f853..2d57e9131 100644 --- a/runtime/src/test/resources/packUnpackAny.baseline +++ b/runtime/src/test/resources/packUnpackAny.baseline @@ -8,6 +8,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {d=seconds: 100 } +> {any=type_url: "type.googleapis.com/google.protobuf.Duration" @@ -25,6 +28,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {message=single_any { type_url: "type.googleapis.com/google.protobuf.Duration" @@ -45,6 +51,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {message=single_any { type_url: "type.googleapis.com/google.protobuf.Duration" @@ -64,6 +73,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {any=single_int64: 1 } @@ -79,12 +91,38 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {any=type_url: "type.googleapis.com/google.protobuf.Int64Value" value: "\b\001" } result: true +Source: list[0] == message +declare any { + value any +} +declare d { + value google.protobuf.Duration +} +declare message { + value dev.cel.testing.testdata.proto3.TestAllTypes +} +declare list { + value list(dyn) +} +=====> +bindings: {list=[type_url: "type.googleapis.com/dev.cel.testing.testdata.proto3.TestAllTypes" +value: "\242\0062\n,type.googleapis.com/google.protobuf.Duration\022\002\bd" +], message=single_any { + type_url: "type.googleapis.com/google.protobuf.Duration" + value: "\bd" +} +} +result: true + Source: TestAllTypes{single_any: d} declare any { value any @@ -95,6 +133,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {d=seconds: 100 } @@ -114,6 +155,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {message=single_int64: -1 } @@ -133,6 +177,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {message=single_uint64: 1 } @@ -152,6 +199,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {} result: single_any { @@ -170,6 +220,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {} result: single_any { @@ -188,6 +241,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {} result: single_any { @@ -206,6 +262,9 @@ declare d { declare message { value dev.cel.testing.testdata.proto3.TestAllTypes } +declare list { + value list(dyn) +} =====> bindings: {message=single_bytes: "happy" } diff --git a/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java b/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java index bc9afa55e..795145508 100644 --- a/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java +++ b/testing/src/main/java/dev/cel/testing/BaseInterpreterTest.java @@ -569,6 +569,7 @@ public void packUnpackAny() throws Exception { declareVariable("any", CelTypes.ANY); declareVariable("d", CelTypes.DURATION); declareVariable("message", CelTypes.createMessage(TestAllTypes.getDescriptor().getFullName())); + declareVariable("list", CelTypes.createList(CelTypes.DYN)); Duration duration = Durations.fromSeconds(100); Any any = Any.pack(duration); TestAllTypes message = TestAllTypes.newBuilder().setSingleAny(any).build(); @@ -584,6 +585,10 @@ public void packUnpackAny() throws Exception { runTest(Activation.of("any", TestAllTypes.newBuilder().setSingleInt64(1).build())); source = "any == 1"; runTest(Activation.of("any", Any.pack(Int64Value.of(1)))); + source = "list[0] == message"; + runTest( + Activation.copyOf( + ImmutableMap.of("list", ImmutableList.of(Any.pack(message)), "message", message))); // pack any source = "TestAllTypes{single_any: d}"; From cd4596a50c388ad62edf3a2f54c828e3c0769abf Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 1 Jul 2024 13:48:25 -0700 Subject: [PATCH 08/33] Compute line offsets once when constructing CelCodePointArray PiperOrigin-RevId: 648474659 --- .../main/java/dev/cel/common/CelSource.java | 22 +++--- .../common/internal/BasicCodePointArray.java | 16 +++- .../common/internal/CelCodePointArray.java | 46 ++++++++++- .../common/internal/EmptyCodePointArray.java | 6 ++ .../common/internal/Latin1CodePointArray.java | 17 +++- .../internal/SupplementalCodePointArray.java | 17 +++- .../cel/common/CelAbstractSyntaxTreeTest.java | 6 +- .../java/dev/cel/common/CelSourceTest.java | 11 +++ .../java/dev/cel/common/internal/BUILD.bazel | 1 + .../internal/CelCodePointArrayTest.java | 77 +++++++++++++++++++ 10 files changed, 186 insertions(+), 33 deletions(-) create mode 100644 common/src/test/java/dev/cel/common/internal/CelCodePointArrayTest.java diff --git a/common/src/main/java/dev/cel/common/CelSource.java b/common/src/main/java/dev/cel/common/CelSource.java index aea607ccd..de779b05b 100644 --- a/common/src/main/java/dev/cel/common/CelSource.java +++ b/common/src/main/java/dev/cel/common/CelSource.java @@ -16,9 +16,9 @@ import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; import com.google.auto.value.AutoValue; -import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; @@ -37,8 +37,6 @@ /** Represents the source content of an expression and related metadata. */ @Immutable public final class CelSource { - private static final Splitter LINE_SPLITTER = Splitter.on('\n'); - private final CelCodePointArray codePoints; private final String description; private final ImmutableList lineOffsets; @@ -194,13 +192,8 @@ public static Builder newBuilder() { } public static Builder newBuilder(String text) { - List lineOffsets = new ArrayList<>(); - int lineOffset = 0; - for (String line : LINE_SPLITTER.split(text)) { - lineOffset += (int) (line.codePoints().count() + 1); - lineOffsets.add(lineOffset); - } - return new Builder(CelCodePointArray.fromString(text), lineOffsets); + CelCodePointArray codePointArray = CelCodePointArray.fromString(text); + return new Builder(codePointArray, codePointArray.lineOffsets()); } /** Builder for {@link CelSource}. */ @@ -212,6 +205,7 @@ public static final class Builder { private final Map macroCalls; private final ImmutableSet.Builder extensions; + private final boolean lineOffsetsAlreadyComputed; private String description; private Builder() { @@ -225,6 +219,7 @@ private Builder(CelCodePointArray codePoints, List lineOffsets) { this.macroCalls = new HashMap<>(); this.extensions = ImmutableSet.builder(); this.description = ""; + this.lineOffsetsAlreadyComputed = !lineOffsets.isEmpty(); } @CanIgnoreReturnValue @@ -236,6 +231,9 @@ public Builder setDescription(String description) { @CanIgnoreReturnValue public Builder addLineOffsets(int lineOffset) { checkArgument(lineOffset >= 0); + checkState( + !lineOffsetsAlreadyComputed, + "Line offsets were already been computed through the provided code points."); lineOffsets.add(lineOffset); return this; } @@ -362,8 +360,8 @@ private LineAndOffset(int line, int offset) { this.offset = offset; } - int line; - int offset; + private final int line; + private final int offset; } /** diff --git a/common/src/main/java/dev/cel/common/internal/BasicCodePointArray.java b/common/src/main/java/dev/cel/common/internal/BasicCodePointArray.java index f4f9ef6c3..251f09d61 100644 --- a/common/src/main/java/dev/cel/common/internal/BasicCodePointArray.java +++ b/common/src/main/java/dev/cel/common/internal/BasicCodePointArray.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkPositionIndexes; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.Immutable; import dev.cel.common.annotations.Internal; @@ -38,21 +39,23 @@ public final class BasicCodePointArray extends CelCodePointArray { private final int offset; private final int size; + private final ImmutableList lineOffsets; - BasicCodePointArray(char[] codePoints, int size) { - this(codePoints, 0, size); + BasicCodePointArray(char[] codePoints, int size, ImmutableList lineOffsets) { + this(codePoints, 0, lineOffsets, size); } - BasicCodePointArray(char[] codePoints, int offset, int size) { + BasicCodePointArray(char[] codePoints, int offset, ImmutableList lineOffsets, int size) { this.codePoints = checkNotNull(codePoints); this.offset = offset; this.size = size; + this.lineOffsets = lineOffsets; } @Override public BasicCodePointArray slice(int i, int j) { checkPositionIndexes(i, j, size()); - return new BasicCodePointArray(codePoints, offset + i, j - i); + return new BasicCodePointArray(codePoints, offset + i, lineOffsets, j - i); } @Override @@ -66,6 +69,11 @@ public int size() { return size; } + @Override + public ImmutableList lineOffsets() { + return lineOffsets; + } + @Override public String toString() { return new String(codePoints, offset, size); diff --git a/common/src/main/java/dev/cel/common/internal/CelCodePointArray.java b/common/src/main/java/dev/cel/common/internal/CelCodePointArray.java index 178b2ac55..94d94b5dc 100644 --- a/common/src/main/java/dev/cel/common/internal/CelCodePointArray.java +++ b/common/src/main/java/dev/cel/common/internal/CelCodePointArray.java @@ -16,6 +16,7 @@ import static com.google.common.base.Strings.isNullOrEmpty; +import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.Immutable; import dev.cel.common.annotations.Internal; import java.util.PrimitiveIterator; @@ -41,6 +42,9 @@ public abstract class CelCodePointArray { /** Returns the number of code points. */ public abstract int size(); + /** Returns the line offsets. */ + public abstract ImmutableList lineOffsets(); + public final int length() { return size(); } @@ -60,8 +64,11 @@ public static CelCodePointArray fromString(String text) { PrimitiveIterator.OfInt codePoints = text.codePoints().iterator(); byte[] byteArray = new byte[text.length()]; int byteIndex = 0; + + LineOffsetContext lineOffsetContext = new LineOffsetContext(); while (codePoints.hasNext()) { int codePoint = codePoints.nextInt(); + lineOffsetContext.process(codePoint); if (codePoint <= 0xff) { byteArray[byteIndex++] = (byte) codePoint; continue; @@ -76,6 +83,7 @@ public static CelCodePointArray fromString(String text) { charArray[charIndex++] = (char) codePoint; while (codePoints.hasNext()) { codePoint = codePoints.nextInt(); + lineOffsetContext.process(codePoint); if (codePoint <= 0xffff) { charArray[charIndex++] = (char) codePoint; continue; @@ -89,11 +97,15 @@ public static CelCodePointArray fromString(String text) { intArray[intIndex++] = codePoint; while (codePoints.hasNext()) { codePoint = codePoints.nextInt(); + lineOffsetContext.process(codePoint); intArray[intIndex++] = codePoint; } - return new SupplementalCodePointArray(intArray, intIndex); + + return new SupplementalCodePointArray( + intArray, intIndex, lineOffsetContext.buildLineOffsets()); } - return new BasicCodePointArray(charArray, charIndex); + + return new BasicCodePointArray(charArray, charIndex, lineOffsetContext.buildLineOffsets()); } int[] intArray = new int[text.length()]; int intIndex = 0; @@ -104,10 +116,36 @@ public static CelCodePointArray fromString(String text) { intArray[intIndex++] = codePoint; while (codePoints.hasNext()) { codePoint = codePoints.nextInt(); + lineOffsetContext.process(codePoint); intArray[intIndex++] = codePoint; } - return new SupplementalCodePointArray(intArray, intIndex); + + return new SupplementalCodePointArray( + intArray, intIndex, lineOffsetContext.buildLineOffsets()); + } + + return new Latin1CodePointArray(byteArray, byteIndex, lineOffsetContext.buildLineOffsets()); + } + + private static class LineOffsetContext { + private static final int NEWLINE_CODE_POINT = 10; + + private final ImmutableList.Builder lineOffsetBuilder; + private int lineOffsetCodePoints; + + private void process(int codePoint) { + lineOffsetCodePoints++; + if (codePoint == NEWLINE_CODE_POINT) { + lineOffsetBuilder.add(lineOffsetCodePoints); + } + } + + private ImmutableList buildLineOffsets() { + return lineOffsetBuilder.add(lineOffsetCodePoints + 1).build(); + } + + private LineOffsetContext() { + this.lineOffsetBuilder = ImmutableList.builder(); } - return new Latin1CodePointArray(byteArray, byteIndex); } } diff --git a/common/src/main/java/dev/cel/common/internal/EmptyCodePointArray.java b/common/src/main/java/dev/cel/common/internal/EmptyCodePointArray.java index 95352d83b..8bca7bf31 100644 --- a/common/src/main/java/dev/cel/common/internal/EmptyCodePointArray.java +++ b/common/src/main/java/dev/cel/common/internal/EmptyCodePointArray.java @@ -14,6 +14,7 @@ package dev.cel.common.internal; +import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.DoNotCall; import com.google.errorprone.annotations.Immutable; import dev.cel.common.annotations.Internal; @@ -55,6 +56,11 @@ public int size() { return 0; } + @Override + public ImmutableList lineOffsets() { + return ImmutableList.of(1); + } + @Override public String toString() { return ""; diff --git a/common/src/main/java/dev/cel/common/internal/Latin1CodePointArray.java b/common/src/main/java/dev/cel/common/internal/Latin1CodePointArray.java index 854df7fa7..a06448aba 100644 --- a/common/src/main/java/dev/cel/common/internal/Latin1CodePointArray.java +++ b/common/src/main/java/dev/cel/common/internal/Latin1CodePointArray.java @@ -20,6 +20,7 @@ import static java.nio.charset.StandardCharsets.ISO_8859_1; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.Immutable; import dev.cel.common.annotations.Internal; @@ -38,21 +39,24 @@ public final class Latin1CodePointArray extends CelCodePointArray { private final int offset; private final int size; + private final ImmutableList lineOffsets; - Latin1CodePointArray(byte[] codePoints, int size) { - this(codePoints, 0, size); + Latin1CodePointArray(byte[] codePoints, int size, ImmutableList lineOffsets) { + this(codePoints, 0, lineOffsets, size); } - Latin1CodePointArray(byte[] codePoints, int offset, int size) { + Latin1CodePointArray( + byte[] codePoints, int offset, ImmutableList lineOffsets, int size) { this.codePoints = checkNotNull(codePoints); this.offset = offset; this.size = size; + this.lineOffsets = lineOffsets; } @Override public Latin1CodePointArray slice(int i, int j) { checkPositionIndexes(i, j, size()); - return new Latin1CodePointArray(codePoints, offset + i, j - i); + return new Latin1CodePointArray(codePoints, offset + i, lineOffsets, j - i); } @Override @@ -66,6 +70,11 @@ public int size() { return size; } + @Override + public ImmutableList lineOffsets() { + return lineOffsets; + } + @Override public String toString() { return new String(codePoints, offset, size, ISO_8859_1); diff --git a/common/src/main/java/dev/cel/common/internal/SupplementalCodePointArray.java b/common/src/main/java/dev/cel/common/internal/SupplementalCodePointArray.java index 8090a591d..30f2fce27 100644 --- a/common/src/main/java/dev/cel/common/internal/SupplementalCodePointArray.java +++ b/common/src/main/java/dev/cel/common/internal/SupplementalCodePointArray.java @@ -19,6 +19,7 @@ import static com.google.common.base.Preconditions.checkPositionIndexes; import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.Immutable; import dev.cel.common.annotations.Internal; @@ -38,21 +39,24 @@ public final class SupplementalCodePointArray extends CelCodePointArray { private final int offset; private final int size; + private final ImmutableList lineOffsets; - SupplementalCodePointArray(int[] codePoints, int size) { - this(codePoints, 0, size); + SupplementalCodePointArray(int[] codePoints, int size, ImmutableList lineOffsets) { + this(codePoints, 0, lineOffsets, size); } - SupplementalCodePointArray(int[] codePoints, int offset, int size) { + SupplementalCodePointArray( + int[] codePoints, int offset, ImmutableList lineOffsets, int size) { this.codePoints = checkNotNull(codePoints); this.offset = offset; this.size = size; + this.lineOffsets = lineOffsets; } @Override public SupplementalCodePointArray slice(int i, int j) { checkPositionIndexes(i, j, size()); - return new SupplementalCodePointArray(codePoints, offset + i, j - i); + return new SupplementalCodePointArray(codePoints, offset + i, lineOffsets, j - i); } @Override @@ -66,6 +70,11 @@ public int size() { return size; } + @Override + public ImmutableList lineOffsets() { + return lineOffsets; + } + @Override public String toString() { return new String(codePoints, offset, size); diff --git a/common/src/test/java/dev/cel/common/CelAbstractSyntaxTreeTest.java b/common/src/test/java/dev/cel/common/CelAbstractSyntaxTreeTest.java index 78c16c165..48e28894a 100644 --- a/common/src/test/java/dev/cel/common/CelAbstractSyntaxTreeTest.java +++ b/common/src/test/java/dev/cel/common/CelAbstractSyntaxTreeTest.java @@ -150,11 +150,7 @@ public void getSource_hasDescriptionEqualToSourceLocation() { public void parsedExpression_createAst() { CelExpr celExpr = CelExpr.newBuilder().setId(1).setConstant(CelConstant.ofValue(2)).build(); CelSource celSource = - CelSource.newBuilder("expression") - .setDescription("desc") - .addPositions(1, 5) - .addLineOffsets(10) - .build(); + CelSource.newBuilder("expression").setDescription("desc").addPositions(1, 5).build(); CelAbstractSyntaxTree ast = CelAbstractSyntaxTree.newParsedAst(celExpr, celSource); diff --git a/common/src/test/java/dev/cel/common/CelSourceTest.java b/common/src/test/java/dev/cel/common/CelSourceTest.java index a97eeb853..d8b3701e3 100644 --- a/common/src/test/java/dev/cel/common/CelSourceTest.java +++ b/common/src/test/java/dev/cel/common/CelSourceTest.java @@ -181,4 +181,15 @@ public void source_withExtension() { .containsExactly(Component.COMPONENT_PARSER, Component.COMPONENT_TYPE_CHECKER); assertThat(celSource.getExtensions()).hasSize(1); } + + @Test + public void source_lineOffsetsAlreadyComputed_throws() { + CelSource.Builder sourceBuilder = CelSource.newBuilder("text"); + + IllegalStateException e = + assertThrows(IllegalStateException.class, () -> sourceBuilder.addLineOffsets(1)); + assertThat(e) + .hasMessageThat() + .contains("Line offsets were already been computed through the provided code points."); + } } diff --git a/common/src/test/java/dev/cel/common/internal/BUILD.bazel b/common/src/test/java/dev/cel/common/internal/BUILD.bazel index 41d98f6c8..af0a673aa 100644 --- a/common/src/test/java/dev/cel/common/internal/BUILD.bazel +++ b/common/src/test/java/dev/cel/common/internal/BUILD.bazel @@ -10,6 +10,7 @@ java_library( srcs = glob(["*.java"]), resources = ["//common/src/test/resources"], deps = [ + "//:auto_value", "//:java_truth", "//common", "//common:options", diff --git a/common/src/test/java/dev/cel/common/internal/CelCodePointArrayTest.java b/common/src/test/java/dev/cel/common/internal/CelCodePointArrayTest.java new file mode 100644 index 000000000..7cb02c5a8 --- /dev/null +++ b/common/src/test/java/dev/cel/common/internal/CelCodePointArrayTest.java @@ -0,0 +1,77 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.common.internal; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import com.google.testing.junit.testparameterinjector.TestParameterValuesProvider; +import java.util.Arrays; +import java.util.List; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public final class CelCodePointArrayTest { + + @Test + public void computeLineOffset( + @TestParameter(valuesProvider = LineOffsetDataProvider.class) LineOffsetTestCase testCase) { + CelCodePointArray codePointArray = CelCodePointArray.fromString(testCase.text()); + + assertThat(codePointArray.lineOffsets()) + .containsExactlyElementsIn(testCase.offsets()) + .inOrder(); + } + + @AutoValue + abstract static class LineOffsetTestCase { + abstract String text(); + + abstract ImmutableList offsets(); + + static LineOffsetTestCase of(String text, Integer... offsets) { + return of(text, Arrays.asList(offsets)); + } + + static LineOffsetTestCase of(String text, List offsets) { + return new AutoValue_CelCodePointArrayTest_LineOffsetTestCase( + text, ImmutableList.copyOf(offsets)); + } + } + + private static final class LineOffsetDataProvider extends TestParameterValuesProvider { + + @Override + protected List provideValues(Context context) { + return Arrays.asList( + // Empty + LineOffsetTestCase.of("", 1), + // ISO-8859-1 + LineOffsetTestCase.of("hello world", 12), + LineOffsetTestCase.of("hello\nworld", 6, 12), + LineOffsetTestCase.of("hello\nworld\n\nfoo\n", 6, 12, 13, 17, 18), + // BMP + LineOffsetTestCase.of("abc 가나다", 8), + LineOffsetTestCase.of("abc\n가나다\n我b很好\n", 4, 8, 13, 14), + // SMP + LineOffsetTestCase.of(" text 가나다 😦😁😑 ", 15), + LineOffsetTestCase.of(" text\n가나다 \n😦😁😑\n\n", 6, 11, 15, 16, 17)); + } + } +} From 039df1f037f1188d17d39e276b444c9646b964bd Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 1 Jul 2024 14:15:19 -0700 Subject: [PATCH 09/33] Add an interface for encapsulating Source properties PiperOrigin-RevId: 648483048 --- common/BUILD.bazel | 6 ++ .../src/main/java/dev/cel/common/BUILD.bazel | 15 +++++ .../main/java/dev/cel/common/CelIssue.java | 2 +- .../main/java/dev/cel/common/CelSource.java | 36 +++-------- .../java/dev/cel/common/CelSourceHelper.java | 61 +++++++++++++++++++ .../src/main/java/dev/cel/common/Source.java | 46 ++++++++++++++ 6 files changed, 136 insertions(+), 30 deletions(-) create mode 100644 common/src/main/java/dev/cel/common/CelSourceHelper.java create mode 100644 common/src/main/java/dev/cel/common/Source.java diff --git a/common/BUILD.bazel b/common/BUILD.bazel index 649c49186..e3d61aad9 100644 --- a/common/BUILD.bazel +++ b/common/BUILD.bazel @@ -59,3 +59,9 @@ java_library( name = "proto_json_adapter", exports = ["//common/src/main/java/dev/cel/common:proto_json_adapter"], ) + +java_library( + name = "source", + visibility = ["//visibility:public"], + exports = ["//common/src/main/java/dev/cel/common:source"], +) diff --git a/common/src/main/java/dev/cel/common/BUILD.bazel b/common/src/main/java/dev/cel/common/BUILD.bazel index 633279a0b..8036dafba 100644 --- a/common/src/main/java/dev/cel/common/BUILD.bazel +++ b/common/src/main/java/dev/cel/common/BUILD.bazel @@ -39,6 +39,7 @@ java_library( ], deps = [ ":error_codes", + ":source", "//:auto_value", "//common/annotations", "//common/ast", @@ -62,6 +63,7 @@ java_library( ], deps = [ ":common", + ":source", "//:auto_value", "//common/annotations", "//common/internal:safe_string_formatter", @@ -175,3 +177,16 @@ java_library( "@maven//:com_google_protobuf_protobuf_java_util", ], ) + +java_library( + name = "source", + srcs = [ + "CelSourceHelper.java", + "Source.java", + ], + deps = [ + "//common/annotations", + "//common/internal", + "@maven//:com_google_guava_guava", + ], +) diff --git a/common/src/main/java/dev/cel/common/CelIssue.java b/common/src/main/java/dev/cel/common/CelIssue.java index 7f7417a53..1e3cdb58e 100644 --- a/common/src/main/java/dev/cel/common/CelIssue.java +++ b/common/src/main/java/dev/cel/common/CelIssue.java @@ -74,7 +74,7 @@ public static CelIssue formatError(int line, int column, String message) { private static final char WIDE_HAT = '\uff3e'; /** Returns a string representing this error that is suitable for displaying to humans. */ - public String toDisplayString(CelSource source) { + public String toDisplayString(Source source) { // Based onhttps://github.com/google/cel-go/blob/v0.5.1/common/error.go#L42. String result = SafeStringFormatter.format( diff --git a/common/src/main/java/dev/cel/common/CelSource.java b/common/src/main/java/dev/cel/common/CelSource.java index de779b05b..b8ffe2598 100644 --- a/common/src/main/java/dev/cel/common/CelSource.java +++ b/common/src/main/java/dev/cel/common/CelSource.java @@ -36,7 +36,8 @@ /** Represents the source content of an expression and related metadata. */ @Immutable -public final class CelSource { +public final class CelSource implements Source { + private final CelCodePointArray codePoints; private final String description; private final ImmutableList lineOffsets; @@ -53,10 +54,12 @@ private CelSource(Builder builder) { this.extensions = checkNotNull(builder.extensions.build()); } + @Override public CelCodePointArray getContent() { return codePoints; } + @Override public String getDescription() { return description; } @@ -107,24 +110,9 @@ public Optional getOffsetLocation(int offset) { return getOffsetLocationImpl(lineOffsets, offset); } - /** - * Get the text from the source expression that corresponds to {@code line}. - * - * @param line the line number starting from 1. - */ + @Override public Optional getSnippet(int line) { - checkArgument(line > 0); - int start = findLineOffset(lineOffsets, line); - if (start == -1) { - return Optional.empty(); - } - int end = findLineOffset(lineOffsets, line + 1); - if (end == -1) { - end = codePoints.size(); - } else { - end--; - } - return Optional.of(end != start ? codePoints.slice(start, end).toString() : ""); + return CelSourceHelper.getSnippet(codePoints, line); } /** @@ -138,7 +126,7 @@ private static Optional getLocationOffsetImpl( List lineOffsets, int line, int column) { checkArgument(line > 0); checkArgument(column >= 0); - int offset = findLineOffset(lineOffsets, line); + int offset = CelSourceHelper.findLineOffset(lineOffsets, line); if (offset == -1) { return Optional.empty(); } @@ -155,16 +143,6 @@ public static Optional getOffsetLocationImpl( return Optional.of(CelSourceLocation.of(lineAndOffset.line, offset - lineAndOffset.offset)); } - private static int findLineOffset(List lineOffsets, int line) { - if (line == 1) { - return 0; - } - if (line > 1 && line <= lineOffsets.size()) { - return lineOffsets.get(line - 2); - } - return -1; - } - private static LineAndOffset findLine(List lineOffsets, int offset) { int line = 1; for (int index = 0; index < lineOffsets.size(); index++) { diff --git a/common/src/main/java/dev/cel/common/CelSourceHelper.java b/common/src/main/java/dev/cel/common/CelSourceHelper.java new file mode 100644 index 000000000..4fde5db7b --- /dev/null +++ b/common/src/main/java/dev/cel/common/CelSourceHelper.java @@ -0,0 +1,61 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.common; + +import static com.google.common.base.Preconditions.checkArgument; + +import com.google.common.collect.ImmutableList; +import dev.cel.common.annotations.Internal; +import dev.cel.common.internal.CelCodePointArray; +import java.util.List; +import java.util.Optional; + +/** + * Helper methods for common source handling in CEL. + * + *

CEL Library Internals. Do Not Use. + */ +@Internal +public final class CelSourceHelper { + + /** Extract the snippet text that corresponds to {@code line}. */ + public static Optional getSnippet(CelCodePointArray content, int line) { + checkArgument(line > 0); + ImmutableList lineOffsets = content.lineOffsets(); + int start = findLineOffset(lineOffsets, line); + if (start == -1) { + return Optional.empty(); + } + int end = findLineOffset(lineOffsets, line + 1); + if (end == -1) { + end = content.size(); + } else { + end--; + } + return Optional.of(end != start ? content.slice(start, end).toString() : ""); + } + + static int findLineOffset(List lineOffsets, int line) { + if (line == 1) { + return 0; + } + if (line > 1 && line <= lineOffsets.size()) { + return lineOffsets.get(line - 2); + } + return -1; + } + + private CelSourceHelper() {} +} diff --git a/common/src/main/java/dev/cel/common/Source.java b/common/src/main/java/dev/cel/common/Source.java new file mode 100644 index 000000000..33a9927a6 --- /dev/null +++ b/common/src/main/java/dev/cel/common/Source.java @@ -0,0 +1,46 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.common; + +import dev.cel.common.annotations.Internal; +import dev.cel.common.internal.CelCodePointArray; +import java.util.Optional; + +/** + * Common interface definition for source properties. + * + *

CEL Library Internals. Do Not Use. Consumers should instead use the canonical implementations + * such as CelSource. + */ +@Internal +public interface Source { + + /** Gets the original textual content of this source, represented in an array of code points. */ + CelCodePointArray getContent(); + + /** + * Gets the description of this source that may optionally be set (example: location of the file + * containing the source). + */ + String getDescription(); + + /** + * Get the text from the source text that corresponds to {@code line}. Snippets are split based on + * the newline ('\n'). + * + * @param line the line number starting from 1. + */ + Optional getSnippet(int line); +} From acc2ff3c602921b09768bbb27b654f32e0ee1e38 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 1 Jul 2024 15:18:57 -0700 Subject: [PATCH 10/33] Add PolicySource PiperOrigin-RevId: 648501376 --- policy/BUILD.bazel | 9 +++ .../src/main/java/dev/cel/policy/BUILD.bazel | 20 +++++++ .../java/dev/cel/policy/CelPolicySource.java | 58 +++++++++++++++++++ .../src/test/java/dev/cel/policy/BUILD.bazel | 24 ++++++++ .../dev/cel/policy/CelPolicySourceTest.java | 48 +++++++++++++++ 5 files changed, 159 insertions(+) create mode 100644 policy/BUILD.bazel create mode 100644 policy/src/main/java/dev/cel/policy/BUILD.bazel create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicySource.java create mode 100644 policy/src/test/java/dev/cel/policy/BUILD.bazel create mode 100644 policy/src/test/java/dev/cel/policy/CelPolicySourceTest.java diff --git a/policy/BUILD.bazel b/policy/BUILD.bazel new file mode 100644 index 000000000..5dfc04ed8 --- /dev/null +++ b/policy/BUILD.bazel @@ -0,0 +1,9 @@ +package( + default_applicable_licenses = ["//:license"], + default_visibility = ["//visibility:public"], +) + +java_library( + name = "policy_source", + exports = ["//policy/src/main/java/dev/cel/policy:policy_source"], +) diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel new file mode 100644 index 000000000..c9c896255 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -0,0 +1,20 @@ +package( + default_applicable_licenses = ["//:license"], + default_visibility = [ + "//policy:__pkg__", + ], +) + +java_library( + name = "policy_source", + srcs = [ + "CelPolicySource.java", + ], + deps = [ + "//:auto_value", + "//common:source", + "//common/internal", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) diff --git a/policy/src/main/java/dev/cel/policy/CelPolicySource.java b/policy/src/main/java/dev/cel/policy/CelPolicySource.java new file mode 100644 index 000000000..b4c110d80 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicySource.java @@ -0,0 +1,58 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import com.google.auto.value.AutoValue; +import com.google.errorprone.annotations.CheckReturnValue; +import dev.cel.common.CelSourceHelper; +import dev.cel.common.Source; +import dev.cel.common.internal.CelCodePointArray; +import java.util.Optional; + +/** CelPolicySource represents the source content of a policy and its related metadata. */ +@AutoValue +public abstract class CelPolicySource implements Source { + + @Override + public abstract CelCodePointArray getContent(); + + @Override + public abstract String getDescription(); + + @Override + public Optional getSnippet(int line) { + return CelSourceHelper.getSnippet(getContent(), line); + } + + /** Builder for {@link CelPolicySource}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setContent(CelCodePointArray content); + + public abstract Builder setDescription(String description); + + @CheckReturnValue + public abstract CelPolicySource build(); + } + + public abstract Builder toBuilder(); + + public static Builder newBuilder(String text) { + return new AutoValue_CelPolicySource.Builder() + .setContent(CelCodePointArray.fromString(text)) + .setDescription(""); + } +} diff --git a/policy/src/test/java/dev/cel/policy/BUILD.bazel b/policy/src/test/java/dev/cel/policy/BUILD.bazel new file mode 100644 index 000000000..26c7996bf --- /dev/null +++ b/policy/src/test/java/dev/cel/policy/BUILD.bazel @@ -0,0 +1,24 @@ +load("//:testing.bzl", "junit4_test_suites") + +package(default_applicable_licenses = ["//:license"]) + +java_library( + name = "tests", + testonly = True, + srcs = glob(["*.java"]), + deps = [ + "//:java_truth", + "//policy:policy_source", + "@maven//:com_google_testparameterinjector_test_parameter_injector", + "@maven//:junit_junit", + ], +) + +junit4_test_suites( + name = "test_suites", + sizes = [ + "small", + ], + src_dir = "src/test/java", + deps = [":tests"], +) diff --git a/policy/src/test/java/dev/cel/policy/CelPolicySourceTest.java b/policy/src/test/java/dev/cel/policy/CelPolicySourceTest.java new file mode 100644 index 000000000..9dc309b3f --- /dev/null +++ b/policy/src/test/java/dev/cel/policy/CelPolicySourceTest.java @@ -0,0 +1,48 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public final class CelPolicySourceTest { + + @Test + public void constructPolicySource_success() { + CelPolicySource policySource = CelPolicySource.newBuilder("hello world").build(); + + assertThat(policySource.getContent().toString()).isEqualTo("hello world"); + assertThat(policySource.getDescription()).isEqualTo(""); + } + + @Test + public void getSnippet_success() { + CelPolicySource policySource = CelPolicySource.newBuilder("hello\nworld").build(); + + assertThat(policySource.getSnippet(1)).hasValue("hello"); + assertThat(policySource.getSnippet(2)).hasValue("world"); + } + + @Test + public void getSnippet_returnsEmpty() { + CelPolicySource policySource = CelPolicySource.newBuilder("hello\nworld").build(); + + assertThat(policySource.getSnippet(3)).isEmpty(); + } +} From 26f25bb78cc0126b37f16aa75d6b351e62cc0518 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 1 Jul 2024 21:51:13 -0700 Subject: [PATCH 11/33] Flag guard the function dispatch result adaptation change PiperOrigin-RevId: 648585602 --- .../src/main/java/dev/cel/common/CelOptions.java | 15 ++++++++++++++- .../java/dev/cel/runtime/DefaultInterpreter.java | 4 +++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/dev/cel/common/CelOptions.java b/common/src/main/java/dev/cel/common/CelOptions.java index f421bee68..fb5ec6ad7 100644 --- a/common/src/main/java/dev/cel/common/CelOptions.java +++ b/common/src/main/java/dev/cel/common/CelOptions.java @@ -93,6 +93,8 @@ public abstract class CelOptions { public abstract int comprehensionMaxIterations(); + public abstract boolean unwrapWellKnownTypesOnFunctionDispatch(); + public abstract Builder toBuilder(); public ImmutableSet toExprFeatures() { @@ -182,7 +184,8 @@ public static Builder newBuilder() { .resolveTypeDependencies(true) .enableUnknownTracking(false) .enableCelValue(false) - .comprehensionMaxIterations(-1); + .comprehensionMaxIterations(-1) + .unwrapWellKnownTypesOnFunctionDispatch(true); } /** @@ -466,6 +469,16 @@ public abstract static class Builder { */ public abstract Builder comprehensionMaxIterations(int value); + /** + * If disabled, CEL runtime will no longer adapt the function dispatch results for protobuf's + * well known types to other types. This option is enabled by default. + * + * @deprecated This will be removed in the future. Please update your codebase to be conformant + * with CEL specification. + */ + @Deprecated + public abstract Builder unwrapWellKnownTypesOnFunctionDispatch(boolean value); + public abstract CelOptions build(); } } diff --git a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java index 80622c62c..babec5dda 100644 --- a/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java +++ b/runtime/src/main/java/dev/cel/runtime/DefaultInterpreter.java @@ -414,7 +414,9 @@ private IntermediateResult evalCall(ExecutionFrame frame, CelExpr expr, CelCall Object dispatchResult = dispatcher.dispatch( metadata, expr.id(), callExpr.function(), reference.overloadIds(), argArray); - dispatchResult = typeProvider.adapt(dispatchResult); + if (celOptions.unwrapWellKnownTypesOnFunctionDispatch()) { + dispatchResult = typeProvider.adapt(dispatchResult); + } return IntermediateResult.create(attr, dispatchResult); } From 2478747d98da268e5e794828232681d6c810a992 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Tue, 2 Jul 2024 10:20:52 -0700 Subject: [PATCH 12/33] Implement YAML parser for policy configs PiperOrigin-RevId: 648760977 --- WORKSPACE | 1 + policy/BUILD.bazel | 27 +- .../src/main/java/dev/cel/policy/BUILD.bazel | 102 +++- .../java/dev/cel/policy/CelPolicyConfig.java | 364 ++++++++++++++ .../dev/cel/policy/CelPolicyConfigParser.java | 31 ++ .../java/dev/cel/policy/CelPolicySource.java | 6 +- .../policy/CelPolicyValidationException.java | 30 ++ .../cel/policy/CelPolicyYamlConfigParser.java | 392 ++++++++++++++++ .../java/dev/cel/policy/ParserContext.java | 33 ++ .../main/java/dev/cel/policy/ValueString.java | 47 ++ .../main/java/dev/cel/policy/YamlHelper.java | 123 +++++ .../dev/cel/policy/YamlParserContextImpl.java | 95 ++++ .../src/test/java/dev/cel/policy/BUILD.bazel | 8 +- .../dev/cel/policy/CelPolicySourceTest.java | 12 +- .../policy/CelPolicyYamlConfigParserTest.java | 443 ++++++++++++++++++ 15 files changed, 1703 insertions(+), 11 deletions(-) create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyConfig.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyConfigParser.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyValidationException.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java create mode 100644 policy/src/main/java/dev/cel/policy/ParserContext.java create mode 100644 policy/src/main/java/dev/cel/policy/ValueString.java create mode 100644 policy/src/main/java/dev/cel/policy/YamlHelper.java create mode 100644 policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java create mode 100644 policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java diff --git a/WORKSPACE b/WORKSPACE index cd72fce2c..09fdb0443 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -75,6 +75,7 @@ maven_install( "org.antlr:antlr4-runtime:" + ANTLR4_VERSION, "org.jspecify:jspecify:0.3.0", "org.threeten:threeten-extra:1.8.0", + "org.yaml:snakeyaml:2.2", ], repositories = [ "https://maven.google.com", diff --git a/policy/BUILD.bazel b/policy/BUILD.bazel index 5dfc04ed8..487bce068 100644 --- a/policy/BUILD.bazel +++ b/policy/BUILD.bazel @@ -1,9 +1,30 @@ package( default_applicable_licenses = ["//:license"], - default_visibility = ["//visibility:public"], + default_visibility = ["//visibility:public"], # TODO: Expose to public ) java_library( - name = "policy_source", - exports = ["//policy/src/main/java/dev/cel/policy:policy_source"], + name = "source", + exports = ["//policy/src/main/java/dev/cel/policy:source"], +) + +java_library( + name = "validation_exception", + exports = ["//policy/src/main/java/dev/cel/policy:validation_exception"], +) + +java_library( + name = "config", + exports = ["//policy/src/main/java/dev/cel/policy:config"], +) + +java_library( + name = "config_parser", + exports = ["//policy/src/main/java/dev/cel/policy:config_parser"], +) + +java_library( + name = "yaml_config_parser", + visibility = ["//visibility:public"], + exports = ["//policy/src/main/java/dev/cel/policy:yaml_config_parser"], ) diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index c9c896255..5fc24e33d 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -6,10 +6,12 @@ package( ) java_library( - name = "policy_source", + name = "source", srcs = [ "CelPolicySource.java", ], + tags = [ + ], deps = [ "//:auto_value", "//common:source", @@ -18,3 +20,101 @@ java_library( "@maven//:com_google_guava_guava", ], ) + +java_library( + name = "validation_exception", + srcs = [ + "CelPolicyValidationException.java", + ], + tags = [ + ], +) + +java_library( + name = "config", + srcs = [ + "CelPolicyConfig.java", + ], + tags = [ + ], + deps = [ + ":source", + "//:auto_value", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "config_parser", + srcs = [ + "CelPolicyConfigParser.java", + ], + tags = [ + ], + deps = [ + ":config", + ":validation_exception", + ], +) + +java_library( + name = "yaml_config_parser", + srcs = [ + "CelPolicyYamlConfigParser.java", + ], + tags = [ + ], + deps = [ + ":config", + ":config_parser", + ":parser_context", + ":policy_common_internal", + ":source", + ":validation_exception", + "//common/internal", + "@maven//:com_google_guava_guava", + "@maven//:org_yaml_snakeyaml", + ], +) + +java_library( + name = "value_string", + srcs = [ + "ValueString.java", + ], + visibility = ["//visibility:private"], + deps = [ + "//:auto_value", + ], +) + +java_library( + name = "parser_context", + srcs = [ + "ParserContext.java", + ], + visibility = ["//visibility:private"], + deps = [ + "//common:source", + ], +) + +java_library( + name = "policy_common_internal", + srcs = [ + "YamlHelper.java", + "YamlParserContextImpl.java", + ], + visibility = ["//visibility:private"], + deps = [ + ":parser_context", + ":value_string", + "//common", + "//common:compiler_common", + "//common:source", + "//common/internal", + "@maven//:com_google_guava_guava", + "@maven//:org_yaml_snakeyaml", + ], +) diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java b/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java new file mode 100644 index 000000000..4d15c4d8b --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java @@ -0,0 +1,364 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.Arrays.stream; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.errorprone.annotations.CheckReturnValue; +import java.util.AbstractMap; +import java.util.Arrays; +import java.util.Optional; + +/** + * CelPolicyConfig represents the policy configuration object to extend the CEL's compilation and + * runtime environment with. + */ +@AutoValue +public abstract class CelPolicyConfig { + + /** Config source. */ + public abstract CelPolicySource configSource(); + + /** Name of the config. */ + public abstract String name(); + + /** + * An optional description of the config (example: location of the file containing the config + * content). + */ + public abstract String description(); + + /** + * Container name to use as the namespace for resolving CEL expression variables and functions. + */ + public abstract String container(); + + /** + * Canonical extensions to enable in the environment, such as Optional, String and Math + * extensions. + */ + public abstract ImmutableSet extensions(); + + /** New variable declarations to add in the compilation environment. */ + public abstract ImmutableSet variables(); + + /** New function declarations to add in the compilation environment. */ + public abstract ImmutableSet functions(); + + /** Builder for {@link CelPolicyConfig}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setConfigSource(CelPolicySource value); + + public abstract Builder setName(String name); + + public abstract Builder setDescription(String description); + + public abstract Builder setContainer(String container); + + public abstract Builder setExtensions(ImmutableSet extensions); + + public abstract Builder setVariables(ImmutableSet variables); + + public abstract Builder setFunctions(ImmutableSet functions); + + @CheckReturnValue + public abstract CelPolicyConfig build(); + } + + /** Creates a new builder to construct a {@link CelPolicyConfig} instance. */ + public static Builder newBuilder() { + return new AutoValue_CelPolicyConfig.Builder() + .setName("") + .setDescription("") + .setContainer("") + .setExtensions(ImmutableSet.of()) + .setVariables(ImmutableSet.of()) + .setFunctions(ImmutableSet.of()); + } + + /** Represents a policy variable declaration. */ + @AutoValue + public abstract static class VariableDecl { + + /** Fully qualified variable name. */ + public abstract String name(); + + /** The type of the variable. */ + public abstract TypeDecl type(); + + /** Builder for {@link VariableDecl}. */ + @AutoValue.Builder + public abstract static class Builder { + + abstract Optional name(); + + abstract Optional type(); + + public abstract Builder setName(String name); + + public abstract Builder setType(TypeDecl typeDecl); + + ImmutableList getMissingRequiredFieldNames() { + return getMissingRequiredFields( + new AbstractMap.SimpleEntry<>("name", name()), + new AbstractMap.SimpleEntry<>("type", type())); + } + + /** Builds a new instance of {@link VariableDecl}. */ + public abstract VariableDecl build(); + } + + public static Builder newBuilder() { + return new AutoValue_CelPolicyConfig_VariableDecl.Builder(); + } + + /** Creates a new builder to construct a {@link VariableDecl} instance. */ + public static VariableDecl create(String name, TypeDecl type) { + return newBuilder().setName(name).setType(type).build(); + } + } + + /** Represents a policy function declaration. */ + @AutoValue + public abstract static class FunctionDecl { + + public abstract String name(); + + public abstract ImmutableSet overloads(); + + /** Builder for {@link FunctionDecl}. */ + @AutoValue.Builder + public abstract static class Builder { + + abstract Optional name(); + + abstract Optional> overloads(); + + public abstract Builder setName(String name); + + public abstract Builder setOverloads(ImmutableSet overloads); + + ImmutableList getMissingRequiredFieldNames() { + return getMissingRequiredFields( + new AbstractMap.SimpleEntry<>("name", name()), + new AbstractMap.SimpleEntry<>("overloads", overloads())); + } + + /** Builds a new instance of {@link FunctionDecl}. */ + public abstract FunctionDecl build(); + } + + /** Creates a new builder to construct a {@link FunctionDecl} instance. */ + public static Builder newBuilder() { + return new AutoValue_CelPolicyConfig_FunctionDecl.Builder(); + } + + /** Creates a new {@link FunctionDecl} with the provided function name and its overloads. */ + public static FunctionDecl create(String name, ImmutableSet overloads) { + return newBuilder().setName(name).setOverloads(overloads).build(); + } + } + + /** Represents an overload declaraion on a policy function. */ + @AutoValue + public abstract static class OverloadDecl { + + /** + * A unique overload ID. Required. This should follow the typical naming convention used in CEL + * (e.g: targetType_func_argType1_argType...) + */ + public abstract String id(); + + /** Target of the function overload if it's a receiver style (example: foo in `foo.f(...)`) */ + public abstract Optional target(); + + /** List of function overload type values. */ + public abstract ImmutableList arguments(); + + /** Return type of the overload. Required. */ + public abstract TypeDecl returnType(); + + /** Builder for {@link OverloadDecl}. */ + @AutoValue.Builder + public abstract static class Builder { + + abstract Optional id(); + + abstract Optional returnType(); + + public abstract Builder setId(String overloadId); + + public abstract Builder setTarget(TypeDecl target); + + // This should stay package-private to encourage add/set methods to be used instead. + abstract ImmutableList.Builder argumentsBuilder(); + + public abstract Builder setArguments(ImmutableList args); + + @CanIgnoreReturnValue + public Builder addArguments(Iterable args) { + this.argumentsBuilder().addAll(checkNotNull(args)); + return this; + } + + @CanIgnoreReturnValue + public Builder addArguments(TypeDecl... args) { + return addArguments(Arrays.asList(args)); + } + + public abstract Builder setReturnType(TypeDecl returnType); + + ImmutableList getMissingRequiredFieldNames() { + return getMissingRequiredFields( + new AbstractMap.SimpleEntry<>("id", id()), + new AbstractMap.SimpleEntry<>("return", returnType())); + } + + /** Builds a new instance of {@link OverloadDecl}. */ + @CheckReturnValue + public abstract OverloadDecl build(); + } + + /** Creates a new builder to construct a {@link OverloadDecl} instance. */ + public static Builder newBuilder() { + return new AutoValue_CelPolicyConfig_OverloadDecl.Builder().setArguments(ImmutableList.of()); + } + } + + /** + * Represents an abstract type declaration used to declare functions and variables in a policy. + */ + @AutoValue + public abstract static class TypeDecl { + + public abstract String name(); + + public abstract ImmutableList params(); + + public abstract Boolean isTypeParam(); + + /** Builder for {@link TypeDecl}. */ + @AutoValue.Builder + public abstract static class Builder { + + abstract Optional name(); + + public abstract Builder setName(String name); + + // This should stay package-private to encourage add/set methods to be used instead. + abstract ImmutableList.Builder paramsBuilder(); + + public abstract Builder setParams(ImmutableList typeDecls); + + @CanIgnoreReturnValue + public Builder addParams(TypeDecl... params) { + return addParams(Arrays.asList(params)); + } + + @CanIgnoreReturnValue + public Builder addParams(Iterable params) { + this.paramsBuilder().addAll(checkNotNull(params)); + return this; + } + + public abstract Builder setIsTypeParam(boolean isTypeParam); + + ImmutableList getMissingRequiredFieldNames() { + return getMissingRequiredFields(new AbstractMap.SimpleEntry<>("type_name", name())); + } + + @CheckReturnValue + public abstract TypeDecl build(); + } + + /** Creates a new {@link TypeDecl} with the provided name. */ + public static TypeDecl create(String name) { + return newBuilder().setName(name).build(); + } + + public static Builder newBuilder() { + return new AutoValue_CelPolicyConfig_TypeDecl.Builder().setIsTypeParam(false); + } + } + + /** + * Represents a configuration for a canonical CEL extension that can be enabled in the + * environment. + */ + @AutoValue + public abstract static class ExtensionConfig { + + /** Name of the extension (ex: bindings, optional, math, etc).". */ + public abstract String name(); + + /** + * Version of the extension. Presently, this field is ignored as CEL-Java extensions are not + * versioned. + */ + public abstract Integer version(); + + /** Builder for {@link ExtensionConfig}. */ + @AutoValue.Builder + public abstract static class Builder { + + abstract Optional name(); + + abstract Optional version(); + + public abstract Builder setName(String name); + + public abstract Builder setVersion(Integer version); + + ImmutableList getMissingRequiredFieldNames() { + return getMissingRequiredFields(new AbstractMap.SimpleEntry<>("name", name())); + } + + /** Builds a new instance of {@link ExtensionConfig}. */ + public abstract ExtensionConfig build(); + } + + /** Creates a new builder to construct a {@link ExtensionConfig} instance. */ + public static Builder newBuilder() { + return new AutoValue_CelPolicyConfig_ExtensionConfig.Builder().setVersion(0); + } + + /** Create a new extension config with the specified name and version set to 0. */ + public static ExtensionConfig of(String name) { + return of(name, 0); + } + + /** Create a new extension config with the specified name and version. */ + public static ExtensionConfig of(String name, int version) { + return newBuilder().setName(name).setVersion(version).build(); + } + } + + @SafeVarargs + private static ImmutableList getMissingRequiredFields( + AbstractMap.SimpleEntry>... requiredFields) { + return stream(requiredFields) + .filter(entry -> !entry.getValue().isPresent()) + .map(AbstractMap.SimpleEntry::getKey) + .collect(toImmutableList()); + } +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyConfigParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyConfigParser.java new file mode 100644 index 000000000..9b03a1dd9 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyConfigParser.java @@ -0,0 +1,31 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +/** Public interface for parsing CEL policy sources. */ +public interface CelPolicyConfigParser { + + /** Parsers the input {@code policyConfigSource} and returns a {@link CelPolicyConfig}. */ + CelPolicyConfig parse(String policyConfigSource) throws CelPolicyValidationException; + + /** + * Parses the input {@code policyConfigSource} and returns a {@link CelPolicyConfig}. + * + *

The {@code description} may be used to help tailor error messages for the location where the + * {@code policySource} originates, e.g. a file name or form UI element. + */ + CelPolicyConfig parse(String policyConfigSource, String description) + throws CelPolicyValidationException; +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicySource.java b/policy/src/main/java/dev/cel/policy/CelPolicySource.java index b4c110d80..75565b563 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicySource.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicySource.java @@ -50,9 +50,9 @@ public abstract static class Builder { public abstract Builder toBuilder(); - public static Builder newBuilder(String text) { + public static Builder newBuilder(CelCodePointArray celCodePointArray) { return new AutoValue_CelPolicySource.Builder() - .setContent(CelCodePointArray.fromString(text)) - .setDescription(""); + .setDescription("") + .setContent(celCodePointArray); } } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyValidationException.java b/policy/src/main/java/dev/cel/policy/CelPolicyValidationException.java new file mode 100644 index 000000000..1f7dc6ac2 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyValidationException.java @@ -0,0 +1,30 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +/** + * CelPolicyValidationException encapsulates all issues that arise when parsing or compiling a + * policy. + */ +public final class CelPolicyValidationException extends Exception { + + CelPolicyValidationException(String message) { + super(message); + } + + CelPolicyValidationException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java new file mode 100644 index 000000000..0cc348cc5 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java @@ -0,0 +1,392 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static dev.cel.policy.YamlHelper.ERROR; +import static dev.cel.policy.YamlHelper.assertRequiredFields; +import static dev.cel.policy.YamlHelper.assertYamlType; +import static dev.cel.policy.YamlHelper.newBoolean; +import static dev.cel.policy.YamlHelper.newInteger; +import static dev.cel.policy.YamlHelper.newString; +import static dev.cel.policy.YamlHelper.parseYamlSource; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import dev.cel.common.internal.CelCodePointArray; +import dev.cel.policy.CelPolicyConfig.ExtensionConfig; +import dev.cel.policy.CelPolicyConfig.FunctionDecl; +import dev.cel.policy.CelPolicyConfig.OverloadDecl; +import dev.cel.policy.CelPolicyConfig.TypeDecl; +import dev.cel.policy.CelPolicyConfig.VariableDecl; +import dev.cel.policy.YamlHelper.YamlNodeType; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; + +/** Package-private class for parsing YAML config files. */ +final class CelPolicyYamlConfigParser implements CelPolicyConfigParser { + // Sentinel values to be returned for various declarations when parsing failure is encountered. + private static final TypeDecl ERROR_TYPE_DECL = TypeDecl.create(ERROR); + private static final VariableDecl ERROR_VARIABLE_DECL = + VariableDecl.create(ERROR, ERROR_TYPE_DECL); + private static final FunctionDecl ERROR_FUNCTION_DECL = + FunctionDecl.create(ERROR, ImmutableSet.of()); + private static final ExtensionConfig ERROR_EXTENSION_DECL = ExtensionConfig.of(ERROR); + + @Override + public CelPolicyConfig parse(String policyConfigSource) throws CelPolicyValidationException { + return parse(policyConfigSource, ""); + } + + @Override + public CelPolicyConfig parse(String policyConfigSource, String description) + throws CelPolicyValidationException { + ParserImpl parser = new ParserImpl(); + + return parser.parseYaml(policyConfigSource, description); + } + + private ImmutableSet parseVariables(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + ImmutableSet.Builder variableSetBuilder = ImmutableSet.builder(); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.LIST)) { + return variableSetBuilder.build(); + } + + SequenceNode variableListNode = (SequenceNode) node; + for (Node elementNode : variableListNode.getValue()) { + variableSetBuilder.add(parseVariable(ctx, elementNode)); + } + + return variableSetBuilder.build(); + } + + private VariableDecl parseVariable(ParserContext ctx, Node node) { + long variableId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, variableId, node, YamlNodeType.MAP)) { + return ERROR_VARIABLE_DECL; + } + + MappingNode variableMap = (MappingNode) node; + VariableDecl.Builder builder = VariableDecl.newBuilder(); + for (NodeTuple nodeTuple : variableMap.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + Node valueNode = nodeTuple.getValueNode(); + String keyName = ((ScalarNode) keyNode).getValue(); + switch (keyName) { + case "name": + builder.setName(newString(ctx, valueNode)); + break; + case "type": + builder.setType(parseTypeDecl(ctx, valueNode)); + break; + default: + ctx.reportError(keyId, String.format("Unsupported variable tag: %s", keyName)); + break; + } + } + + if (!assertRequiredFields(ctx, variableId, builder.getMissingRequiredFieldNames())) { + return ERROR_VARIABLE_DECL; + } + + return builder.build(); + } + + private ImmutableSet parseFunctions(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + ImmutableSet.Builder functionSetBuilder = ImmutableSet.builder(); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.LIST)) { + return functionSetBuilder.build(); + } + + SequenceNode functionListNode = (SequenceNode) node; + for (Node elementNode : functionListNode.getValue()) { + functionSetBuilder.add(parseFunction(ctx, elementNode)); + } + + return functionSetBuilder.build(); + } + + private FunctionDecl parseFunction(ParserContext ctx, Node node) { + long functionId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, functionId, node, YamlNodeType.MAP)) { + return ERROR_FUNCTION_DECL; + } + + MappingNode functionMap = (MappingNode) node; + FunctionDecl.Builder builder = FunctionDecl.newBuilder(); + for (NodeTuple nodeTuple : functionMap.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + Node valueNode = nodeTuple.getValueNode(); + String keyName = ((ScalarNode) keyNode).getValue(); + switch (keyName) { + case "name": + builder.setName(newString(ctx, valueNode)); + break; + case "overloads": + builder.setOverloads(parseOverloads(ctx, valueNode)); + break; + default: + ctx.reportError(keyId, String.format("Unsupported function tag: %s", keyName)); + break; + } + } + + if (!assertRequiredFields(ctx, functionId, builder.getMissingRequiredFieldNames())) { + return ERROR_FUNCTION_DECL; + } + + return builder.build(); + } + + private static ImmutableSet parseOverloads(ParserContext ctx, Node node) { + long listId = ctx.collectMetadata(node); + ImmutableSet.Builder overloadSetBuilder = ImmutableSet.builder(); + if (!assertYamlType(ctx, listId, node, YamlNodeType.LIST)) { + return overloadSetBuilder.build(); + } + + SequenceNode overloadListNode = (SequenceNode) node; + for (Node overloadMapNode : overloadListNode.getValue()) { + long overloadMapId = ctx.collectMetadata(overloadMapNode); + if (!assertYamlType(ctx, overloadMapId, overloadMapNode, YamlNodeType.MAP)) { + continue; + } + + MappingNode mapNode = (MappingNode) overloadMapNode; + OverloadDecl.Builder overloadDeclBuilder = OverloadDecl.newBuilder(); + for (NodeTuple nodeTuple : mapNode.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + if (!assertYamlType(ctx, keyId, keyNode, YamlNodeType.STRING, YamlNodeType.TEXT)) { + continue; + } + + Node valueNode = nodeTuple.getValueNode(); + String fieldName = ((ScalarNode) keyNode).getValue(); + switch (fieldName) { + case "id": + overloadDeclBuilder.setId(newString(ctx, valueNode)); + break; + case "args": + overloadDeclBuilder.addArguments(parseOverloadArguments(ctx, valueNode)); + break; + case "return": + overloadDeclBuilder.setReturnType(parseTypeDecl(ctx, valueNode)); + break; + case "target": + overloadDeclBuilder.setTarget(parseTypeDecl(ctx, valueNode)); + break; + default: + ctx.reportError(keyId, String.format("Unsupported overload tag: %s", fieldName)); + break; + } + } + + if (assertRequiredFields( + ctx, overloadMapId, overloadDeclBuilder.getMissingRequiredFieldNames())) { + overloadSetBuilder.add(overloadDeclBuilder.build()); + } + } + + return overloadSetBuilder.build(); + } + + private static ImmutableList parseOverloadArguments( + ParserContext ctx, Node node) { + long listValueId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, listValueId, node, YamlNodeType.LIST)) { + return ImmutableList.of(); + } + SequenceNode paramsListNode = (SequenceNode) node; + ImmutableList.Builder builder = ImmutableList.builder(); + for (Node elementNode : paramsListNode.getValue()) { + builder.add(parseTypeDecl(ctx, elementNode)); + } + + return builder.build(); + } + + private static ImmutableSet parseExtensions(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + ImmutableSet.Builder extensionConfigBuilder = ImmutableSet.builder(); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.LIST)) { + return extensionConfigBuilder.build(); + } + + SequenceNode extensionListNode = (SequenceNode) node; + for (Node elementNode : extensionListNode.getValue()) { + extensionConfigBuilder.add(parseExtension(ctx, elementNode)); + } + + return extensionConfigBuilder.build(); + } + + private static ExtensionConfig parseExtension(ParserContext ctx, Node node) { + long extensionId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, extensionId, node, YamlNodeType.MAP)) { + return ERROR_EXTENSION_DECL; + } + + MappingNode extensionMap = (MappingNode) node; + ExtensionConfig.Builder builder = ExtensionConfig.newBuilder(); + for (NodeTuple nodeTuple : extensionMap.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + Node valueNode = nodeTuple.getValueNode(); + String keyName = ((ScalarNode) keyNode).getValue(); + switch (keyName) { + case "name": + builder.setName(newString(ctx, valueNode)); + break; + case "version": + builder.setVersion(newInteger(ctx, valueNode)); + break; + default: + ctx.reportError(keyId, String.format("Unsupported extension tag: %s", keyName)); + break; + } + } + + if (!assertRequiredFields(ctx, extensionId, builder.getMissingRequiredFieldNames())) { + return ERROR_EXTENSION_DECL; + } + + return builder.build(); + } + + private static TypeDecl parseTypeDecl(ParserContext ctx, Node node) { + TypeDecl.Builder builder = TypeDecl.newBuilder(); + long id = ctx.collectMetadata(node); + if (!assertYamlType(ctx, id, node, YamlNodeType.MAP)) { + return ERROR_TYPE_DECL; + } + + MappingNode mapNode = (MappingNode) node; + for (NodeTuple nodeTuple : mapNode.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + if (!assertYamlType(ctx, keyId, keyNode, YamlNodeType.STRING, YamlNodeType.TEXT)) { + continue; + } + + Node valueNode = nodeTuple.getValueNode(); + String fieldName = ((ScalarNode) keyNode).getValue(); + switch (fieldName) { + case "type_name": + builder.setName(newString(ctx, valueNode)); + break; + case "is_type_param": + builder.setIsTypeParam(newBoolean(ctx, valueNode)); + break; + case "params": + long listValueId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, listValueId, valueNode, YamlNodeType.LIST)) { + break; + } + SequenceNode paramsListNode = (SequenceNode) valueNode; + for (Node elementNode : paramsListNode.getValue()) { + builder.addParams(parseTypeDecl(ctx, elementNode)); + } + break; + default: + ctx.reportError(keyId, String.format("Unsupported type decl tag: %s", fieldName)); + break; + } + } + + if (!assertRequiredFields(ctx, id, builder.getMissingRequiredFieldNames())) { + return ERROR_TYPE_DECL; + } + + return builder.build(); + } + + private class ParserImpl { + + private CelPolicyConfig parseYaml(String source, String description) + throws CelPolicyValidationException { + Node node; + try { + node = parseYamlSource(source); + } catch (RuntimeException e) { + throw new CelPolicyValidationException("YAML document is malformed: " + e.getMessage(), e); + } + + CelCodePointArray configCodePointArray = CelCodePointArray.fromString(source); + ParserContext ctx = YamlParserContextImpl.newInstance(configCodePointArray); + CelPolicyConfig.Builder policyConfig = parseConfig(ctx, node); + CelPolicySource configSource = + CelPolicySource.newBuilder(configCodePointArray).setDescription(description).build(); + if (ctx.hasError()) { + throw new CelPolicyValidationException(ctx.getIssueString(configSource)); + } + return policyConfig.setConfigSource(configSource).build(); + } + + private CelPolicyConfig.Builder parseConfig(ParserContext ctx, Node node) { + CelPolicyConfig.Builder builder = CelPolicyConfig.newBuilder(); + long id = ctx.collectMetadata(node); + if (!assertYamlType(ctx, id, node, YamlNodeType.MAP)) { + return builder; + } + MappingNode rootNode = (MappingNode) node; + for (NodeTuple nodeTuple : rootNode.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + if (!assertYamlType(ctx, keyId, keyNode, YamlNodeType.STRING, YamlNodeType.TEXT)) { + continue; + } + Node valueNode = nodeTuple.getValueNode(); + String fieldName = ((ScalarNode) keyNode).getValue(); + switch (fieldName) { + case "name": + builder.setName(newString(ctx, valueNode)); + break; + case "description": + builder.setDescription(newString(ctx, valueNode)); + break; + case "container": + builder.setContainer(newString(ctx, valueNode)); + break; + case "variables": + builder.setVariables(parseVariables(ctx, valueNode)); + break; + case "functions": + builder.setFunctions(parseFunctions(ctx, valueNode)); + break; + case "extensions": + builder.setExtensions(parseExtensions(ctx, valueNode)); + break; + default: + ctx.reportError(id, "Unknown config tag: " + fieldName); + // continue handling the rest of the nodes + } + } + return builder; + } + } + + static CelPolicyYamlConfigParser newInstance() { + return new CelPolicyYamlConfigParser(); + } + + private CelPolicyYamlConfigParser() {} +} diff --git a/policy/src/main/java/dev/cel/policy/ParserContext.java b/policy/src/main/java/dev/cel/policy/ParserContext.java new file mode 100644 index 000000000..ec4c72f20 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/ParserContext.java @@ -0,0 +1,33 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import dev.cel.common.Source; + +/** + * ParserContext declares a set of interfaces for creating and managing metadata for parsed + * policies. + */ +public interface ParserContext { + long nextId(); + + long collectMetadata(T node); + + void reportError(long id, String message); + + String getIssueString(Source source); + + boolean hasError(); +} diff --git a/policy/src/main/java/dev/cel/policy/ValueString.java b/policy/src/main/java/dev/cel/policy/ValueString.java new file mode 100644 index 000000000..b1de15be5 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/ValueString.java @@ -0,0 +1,47 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import com.google.auto.value.AutoValue; + +/** ValueString contains an identifier corresponding to source metadata and a simple string. */ +@AutoValue +public abstract class ValueString { + + /** A unique identifier. This is populated by the parser. */ + abstract long id(); + + abstract String value(); + + @AutoValue.Builder + abstract static class Builder { + + abstract Builder setId(long id); + + abstract Builder setValue(String value); + + abstract ValueString build(); + } + + /** Builder for {@link ValueString}. */ + static Builder newBuilder() { + return new AutoValue_ValueString.Builder().setId(0).setValue(""); + } + + /** Creates a new {@link ValueString} instance with the specified ID and string value. */ + static ValueString of(long id, String value) { + return newBuilder().setId(id).setValue(value).build(); + } +} diff --git a/policy/src/main/java/dev/cel/policy/YamlHelper.java b/policy/src/main/java/dev/cel/policy/YamlHelper.java new file mode 100644 index 000000000..a29e8c73b --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/YamlHelper.java @@ -0,0 +1,123 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static java.util.Arrays.stream; +import static java.util.stream.Collectors.joining; + +import com.google.common.base.Joiner; +import java.io.StringReader; +import java.util.List; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.ScalarNode; + +final class YamlHelper { + static final String ERROR = "*error*"; + + enum YamlNodeType { + MAP("tag:yaml.org,2002:map"), + STRING("tag:yaml.org,2002:str"), + BOOLEAN("tag:yaml.org,2002:bool"), + INTEGER("tag:yaml.org,2002:int"), + DOUBLE("tag:yaml.org,2002:float"), + TEXT("!txt"), + LIST("tag:yaml.org,2002:seq"), + ; + + private final String tag; + + String tag() { + return tag; + } + + YamlNodeType(String tag) { + this.tag = tag; + } + } + + static Node parseYamlSource(String policyContent) { + Yaml yaml = new Yaml(new SafeConstructor(new LoaderOptions())); + + return yaml.compose(new StringReader(policyContent)); + } + + static boolean assertRequiredFields( + ParserContext ctx, long id, List missingRequiredFields) { + if (missingRequiredFields.isEmpty()) { + return true; + } + + ctx.reportError( + id, + String.format( + "Missing required attribute(s): %s", Joiner.on(", ").join(missingRequiredFields))); + return false; + } + + static boolean assertYamlType( + ParserContext ctx, long id, Node node, YamlNodeType... expectedNodeTypes) { + String nodeTag = node.getTag().getValue(); + for (YamlNodeType expectedNodeType : expectedNodeTypes) { + if (expectedNodeType.tag().equals(nodeTag)) { + return true; + } + } + ctx.reportError( + id, + String.format( + "Got yaml node type %s, wanted type(s) [%s]", + nodeTag, stream(expectedNodeTypes).map(YamlNodeType::tag).collect(joining(" ")))); + return false; + } + + static Integer newInteger(ParserContext ctx, Node node) { + long id = ctx.collectMetadata(node); + if (!assertYamlType(ctx, id, node, YamlNodeType.INTEGER)) { + return 0; + } + + return Integer.parseInt(((ScalarNode) node).getValue()); + } + + static boolean newBoolean(ParserContext ctx, Node node) { + long id = ctx.collectMetadata(node); + if (!assertYamlType(ctx, id, node, YamlNodeType.BOOLEAN)) { + return false; + } + + return Boolean.parseBoolean(((ScalarNode) node).getValue()); + } + + static String newString(ParserContext ctx, Node node) { + return YamlHelper.newValueString(ctx, node).value(); + } + + static ValueString newValueString(ParserContext ctx, Node node) { + long id = ctx.collectMetadata(node); + if (!assertYamlType(ctx, id, node, YamlNodeType.STRING, YamlNodeType.TEXT)) { + return ValueString.of(id, ERROR); + } + + ScalarNode scalarNode = (ScalarNode) node; + + // TODO: Compute relative source for multiline strings + return ValueString.of(id, scalarNode.getValue()); + } + + private YamlHelper() {} +} diff --git a/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java new file mode 100644 index 000000000..34f70db6c --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java @@ -0,0 +1,95 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.common.base.Joiner; +import dev.cel.common.CelIssue; +import dev.cel.common.CelSourceLocation; +import dev.cel.common.Source; +import dev.cel.common.internal.CelCodePointArray; +import java.util.ArrayList; +import java.util.HashMap; +import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.ScalarNode; + +/** Package-private class to assist with storing policy parsing context. */ +final class YamlParserContextImpl implements ParserContext { + + private static final Joiner JOINER = Joiner.on('\n'); + + private final ArrayList issues; + private final HashMap idToLocationMap; + private final HashMap idToOffsetMap; + private final CelCodePointArray policyContent; + private long id; + + @Override + public void reportError(long id, String message) { + issues.add(CelIssue.formatError(idToLocationMap.get(id), message)); + } + + @Override + public String getIssueString(Source source) { + return JOINER.join( + issues.stream().map(iss -> iss.toDisplayString(source)).collect(toImmutableList())); + } + + @Override + public boolean hasError() { + return !issues.isEmpty(); + } + + @Override + public long collectMetadata(Node node) { + long id = nextId(); + int line = node.getStartMark().getLine() + 1; // Yaml lines are 0 indexed + int column = node.getStartMark().getColumn(); + if (node instanceof ScalarNode) { + DumperOptions.ScalarStyle style = ((ScalarNode) node).getScalarStyle(); + if (style.equals(DumperOptions.ScalarStyle.SINGLE_QUOTED) + || style.equals(DumperOptions.ScalarStyle.DOUBLE_QUOTED)) { + column++; + } + } + idToLocationMap.put(id, CelSourceLocation.of(line, column)); + + int offset = 0; + if (line > 1) { + offset = policyContent.lineOffsets().get(line - 2) + column; + } + idToOffsetMap.put(id, offset); + + return id; + } + + @Override + public long nextId() { + return ++id; + } + + static ParserContext newInstance(CelCodePointArray policyContent) { + return new YamlParserContextImpl(policyContent); + } + + private YamlParserContextImpl(CelCodePointArray policyContent) { + this.issues = new ArrayList<>(); + this.idToLocationMap = new HashMap<>(); + this.idToOffsetMap = new HashMap<>(); + this.policyContent = policyContent; + } +} diff --git a/policy/src/test/java/dev/cel/policy/BUILD.bazel b/policy/src/test/java/dev/cel/policy/BUILD.bazel index 26c7996bf..951b45f6a 100644 --- a/policy/src/test/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/test/java/dev/cel/policy/BUILD.bazel @@ -8,7 +8,13 @@ java_library( srcs = glob(["*.java"]), deps = [ "//:java_truth", - "//policy:policy_source", + "//common/internal", + "//policy:config", + "//policy:config_parser", + "//policy:source", + "//policy:validation_exception", + "//policy:yaml_config_parser", + "@maven//:com_google_guava_guava", "@maven//:com_google_testparameterinjector_test_parameter_injector", "@maven//:junit_junit", ], diff --git a/policy/src/test/java/dev/cel/policy/CelPolicySourceTest.java b/policy/src/test/java/dev/cel/policy/CelPolicySourceTest.java index 9dc309b3f..9aa73ee6c 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicySourceTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicySourceTest.java @@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.common.internal.CelCodePointArray; import org.junit.Test; import org.junit.runner.RunWith; @@ -25,7 +26,10 @@ public final class CelPolicySourceTest { @Test public void constructPolicySource_success() { - CelPolicySource policySource = CelPolicySource.newBuilder("hello world").build(); + CelPolicySource policySource = + CelPolicySource.newBuilder(CelCodePointArray.fromString("hello world")) + .setDescription("") + .build(); assertThat(policySource.getContent().toString()).isEqualTo("hello world"); assertThat(policySource.getDescription()).isEqualTo(""); @@ -33,7 +37,8 @@ public void constructPolicySource_success() { @Test public void getSnippet_success() { - CelPolicySource policySource = CelPolicySource.newBuilder("hello\nworld").build(); + CelPolicySource policySource = + CelPolicySource.newBuilder(CelCodePointArray.fromString("hello\nworld")).build(); assertThat(policySource.getSnippet(1)).hasValue("hello"); assertThat(policySource.getSnippet(2)).hasValue("world"); @@ -41,7 +46,8 @@ public void getSnippet_success() { @Test public void getSnippet_returnsEmpty() { - CelPolicySource policySource = CelPolicySource.newBuilder("hello\nworld").build(); + CelPolicySource policySource = + CelPolicySource.newBuilder(CelCodePointArray.fromString("hello\nworld")).build(); assertThat(policySource.getSnippet(3)).isEmpty(); } diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java new file mode 100644 index 000000000..020d9f689 --- /dev/null +++ b/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java @@ -0,0 +1,443 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableSet; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.policy.CelPolicyConfig.ExtensionConfig; +import dev.cel.policy.CelPolicyConfig.FunctionDecl; +import dev.cel.policy.CelPolicyConfig.OverloadDecl; +import dev.cel.policy.CelPolicyConfig.TypeDecl; +import dev.cel.policy.CelPolicyConfig.VariableDecl; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public final class CelPolicyYamlConfigParserTest { + + private static final CelPolicyConfigParser POLICY_CONFIG_PARSER = + CelPolicyYamlConfigParser.newInstance(); + + @Test + public void config_setBasicProperties() throws Exception { + String yamlConfig = "name: hello\n" + "description: empty\n" + "container: pb.pkg\n"; + + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(yamlConfig); + + assertThat(policyConfig) + .isEqualTo( + CelPolicyConfig.newBuilder() + .setConfigSource(policyConfig.configSource()) + .setName("hello") + .setDescription("empty") + .setContainer("pb.pkg") + .build()); + } + + @Test + public void config_setExtensions() throws Exception { + String yamlConfig = + "extensions:\n" + + " - name: 'bindings'\n" + + " - name: 'encoders'\n" + + " - name: 'math'\n" + + " - name: 'optional'\n" + + " - name: 'protos'\n" + + " - name: 'sets'\n" + + " - name: 'strings'\n" + + " version: 1"; + + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(yamlConfig); + + assertThat(policyConfig) + .isEqualTo( + CelPolicyConfig.newBuilder() + .setConfigSource(policyConfig.configSource()) + .setExtensions( + ImmutableSet.of( + ExtensionConfig.of("bindings"), + ExtensionConfig.of("encoders"), + ExtensionConfig.of("math"), + ExtensionConfig.of("optional"), + ExtensionConfig.of("protos"), + ExtensionConfig.of("sets"), + ExtensionConfig.of("strings", 1))) + .build()); + } + + @Test + public void config_setFunctions() throws Exception { + String yamlConfig = + "functions:\n" + + " - name: 'coalesce'\n" + + " overloads:\n" + + " - id: 'null_coalesce_int'\n" + + " target:\n" + + " type_name: 'null_type'\n" + + " args:\n" + + " - type_name: 'int'\n" + + " return:\n" + + " type_name: 'int'\n" + + " - id: 'coalesce_null_int'\n" + + " args:\n" + + " - type_name: 'null_type'\n" + + " - type_name: 'int'\n" + + " return:\n" + + " type_name: 'int' \n" + + " - id: 'int_coalesce_int'\n" + + " target: \n" + + " type_name: 'int'\n" + + " args:\n" + + " - type_name: 'int'\n" + + " return: \n" + + " type_name: 'int'\n" + + " - id: 'optional_T_coalesce_T'\n" + + " target: \n" + + " type_name: 'optional_type'\n" + + " params:\n" + + " - type_name: 'T'\n" + + " is_type_param: true\n" + + " args:\n" + + " - type_name: 'T'\n" + + " is_type_param: true\n" + + " return: \n" + + " type_name: 'T'\n" + + " is_type_param: true"; + + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(yamlConfig); + + assertThat(policyConfig) + .isEqualTo( + CelPolicyConfig.newBuilder() + .setConfigSource(policyConfig.configSource()) + .setFunctions( + ImmutableSet.of( + FunctionDecl.create( + "coalesce", + ImmutableSet.of( + OverloadDecl.newBuilder() + .setId("null_coalesce_int") + .setTarget(TypeDecl.create("null_type")) + .addArguments(TypeDecl.create("int")) + .setReturnType(TypeDecl.create("int")) + .build(), + OverloadDecl.newBuilder() + .setId("coalesce_null_int") + .addArguments( + TypeDecl.create("null_type"), TypeDecl.create("int")) + .setReturnType(TypeDecl.create("int")) + .build(), + OverloadDecl.newBuilder() + .setId("int_coalesce_int") + .setTarget(TypeDecl.create("int")) + .addArguments(TypeDecl.create("int")) + .setReturnType(TypeDecl.create("int")) + .build(), + OverloadDecl.newBuilder() + .setId("optional_T_coalesce_T") + .setTarget( + TypeDecl.newBuilder() + .setName("optional_type") + .addParams( + TypeDecl.newBuilder() + .setName("T") + .setIsTypeParam(true) + .build()) + .build()) + .addArguments( + TypeDecl.newBuilder() + .setName("T") + .setIsTypeParam(true) + .build()) + .setReturnType( + TypeDecl.newBuilder() + .setName("T") + .setIsTypeParam(true) + .build()) + .build())))) + .build()); + } + + @Test + public void config_setListVariable() throws Exception { + String yamlConfig = + "variables:\n" + + "- name: 'request'\n" + + " type:\n" + + " type_name: 'list'\n" + + " params:\n" + + " - type_name: 'string'"; + + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(yamlConfig); + + assertThat(policyConfig) + .isEqualTo( + CelPolicyConfig.newBuilder() + .setConfigSource(policyConfig.configSource()) + .setVariables( + ImmutableSet.of( + VariableDecl.create( + "request", + TypeDecl.newBuilder() + .setName("list") + .addParams(TypeDecl.create("string")) + .build()))) + .build()); + } + + @Test + public void config_setMapVariable() throws Exception { + String yamlConfig = + "variables:\n" + + "- name: 'request'\n" + + " type:\n" + + " type_name: 'map'\n" + + " params:\n" + + " - type_name: 'string'\n" + + " - type_name: 'dyn'"; + + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(yamlConfig); + + assertThat(policyConfig) + .isEqualTo( + CelPolicyConfig.newBuilder() + .setConfigSource(policyConfig.configSource()) + .setVariables( + ImmutableSet.of( + VariableDecl.create( + "request", + TypeDecl.newBuilder() + .setName("map") + .addParams(TypeDecl.create("string"), TypeDecl.create("dyn")) + .build()))) + .build()); + } + + @Test + public void config_setMessageVariable() throws Exception { + String yamlConfig = + "variables:\n" + + "- name: 'request'\n" + + " type:\n" + + " type_name: 'google.rpc.context.AttributeContext.Request'"; + + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(yamlConfig); + + assertThat(policyConfig) + .isEqualTo( + CelPolicyConfig.newBuilder() + .setConfigSource(policyConfig.configSource()) + .setVariables( + ImmutableSet.of( + VariableDecl.create( + "request", + TypeDecl.create("google.rpc.context.AttributeContext.Request")))) + .build()); + } + + @Test + public void config_parseErrors(@TestParameter ConfigParseErrorTestcase testCase) { + CelPolicyValidationException e = + assertThrows( + CelPolicyValidationException.class, + () -> POLICY_CONFIG_PARSER.parse(testCase.yamlConfig)); + assertThat(e).hasMessageThat().isEqualTo(testCase.expectedErrorMessage); + } + + // Note: dangling comments in expressions below is to retain the newlines by preventing auto + // formatter from compressing them in a single line. + private enum ConfigParseErrorTestcase { + MALFORMED_YAML_DOCUMENT( + "a:\na", + "YAML document is malformed: while scanning a simple key\n" + + " in 'reader', line 2, column 1:\n" + + " a\n" + + " ^\n" + + "could not find expected ':'\n" + + " in 'reader', line 2, column 2:\n" + + " a\n" + + " ^\n"), + ILLEGAL_YAML_TYPE_CONFIG_KEY( + "1: test", + "ERROR: :1:1: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:str !txt]\n" + + " | 1: test\n" + + " | ^"), + ILLEGAL_YAML_TYPE_CONFIG_VALUE( + "test: 1", "ERROR: :1:1: Unknown config tag: test\n" + " | test: 1\n" + " | ^"), + ILLEGAL_YAML_TYPE_VARIABLE_LIST( + "variables: 1", + "ERROR: :1:12: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:seq]\n" + + " | variables: 1\n" + + " | ...........^"), + ILLEGAL_YAML_TYPE_VARIABLE_VALUE( + "variables:\n - 1", + "ERROR: :2:4: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - 1\n" + + " | ...^"), + ILLEGAL_YAML_TYPE_FUNCTION_LIST( + "functions: 1", + "ERROR: :1:12: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:seq]\n" + + " | functions: 1\n" + + " | ...........^"), + ILLEGAL_YAML_TYPE_FUNCTION_VALUE( + "functions:\n - 1", + "ERROR: :2:4: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - 1\n" + + " | ...^"), + ILLEGAL_YAML_TYPE_OVERLOAD_LIST( + "functions:\n" // + + " - name: foo\n" // + + " overloads: 1", + "ERROR: :3:15: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:seq]\n" + + " | overloads: 1\n" + + " | ..............^"), + ILLEGAL_YAML_TYPE_OVERLOAD_VALUE( + "functions:\n" // + + " - name: foo\n" // + + " overloads:\n" // + + " - 2", + "ERROR: :4:7: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - 2\n" + + " | ......^"), + ILLEGAL_YAML_TYPE_OVERLOAD_VALUE_MAP_KEY( + "functions:\n" // + + " - name: foo\n" // + + " overloads:\n" // + + " - 2: test", + "ERROR: :4:9: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:str !txt]\n" + + " | - 2: test\n" + + " | ........^\n" + + "ERROR: :4:9: Missing required attribute(s): id, return\n" + + " | - 2: test\n" + + " | ........^"), + ILLEGAL_YAML_TYPE_EXTENSION_LIST( + "extensions: 1", + "ERROR: :1:13: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:seq]\n" + + " | extensions: 1\n" + + " | ............^"), + ILLEGAL_YAML_TYPE_EXTENSION_VALUE( + "extensions:\n - 1", + "ERROR: :2:4: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - 1\n" + + " | ...^"), + ILLEGAL_YAML_TYPE_TYPE_DECL( + "variables:\n" // + + " - name: foo\n" // + + " type: 1", + "ERROR: :3:10: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | type: 1\n" + + " | .........^"), + ILLEGAL_YAML_TYPE_TYPE_VALUE( + "variables:\n" + + " - name: foo\n" + + " type:\n" + + " type_name: bar\n" + + " 1: hello", + "ERROR: :5:6: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:str !txt]\n" + + " | 1: hello\n" + + " | .....^"), + ILLEGAL_YAML_TYPE_TYPE_PARAMS_LIST( + "variables:\n" + + " - name: foo\n" + + " type:\n" + + " type_name: bar\n" + + " params: 1", + "ERROR: :4:6: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:seq]\n" + + " | type_name: bar\n" + + " | .....^"), + UNSUPPORTED_CONFIG_TAG( + "unsupported: test", + "ERROR: :1:1: Unknown config tag: unsupported\n" + + " | unsupported: test\n" + + " | ^"), + UNSUPPORTED_EXTENSION_TAG( + "extensions:\n" // + + " - name: foo\n" // + + " unsupported: test", + "ERROR: :3:5: Unsupported extension tag: unsupported\n" + + " | unsupported: test\n" + + " | ....^"), + UNSUPPORTED_TYPE_DECL_TAG( + "variables:\n" + + "- name: foo\n" + + " type:\n" + + " type_name: bar\n" + + " unsupported: hello", + "ERROR: :5:6: Unsupported type decl tag: unsupported\n" + + " | unsupported: hello\n" + + " | .....^"), + MISSING_VARIABLE_PROPERTIES( + "variables:\n - illegal: 2", + "ERROR: :2:4: Unsupported variable tag: illegal\n" + + " | - illegal: 2\n" + + " | ...^\n" + + "ERROR: :2:4: Missing required attribute(s): name, type\n" + + " | - illegal: 2\n" + + " | ...^"), + MISSING_OVERLOAD_RETURN( + "functions:\n" + + " - name: 'missing_return'\n" + + " overloads:\n" + + " - id: 'zero_arity'\n", + "ERROR: :4:9: Missing required attribute(s): return\n" + + " | - id: 'zero_arity'\n" + + " | ........^"), + MISSING_FUNCTION_NAME( + "functions:\n" + + " - overloads:\n" + + " - id: 'foo'\n" + + " return:\n" + + " type_name: 'string'\n", + "ERROR: :2:5: Missing required attribute(s): name\n" + + " | - overloads:\n" + + " | ....^"), + MISSING_OVERLOAD( + "functions:\n" + " - name: 'missing_overload'\n", + "ERROR: :2:5: Missing required attribute(s): overloads\n" + + " | - name: 'missing_overload'\n" + + " | ....^"), + MISSING_EXTENSION_NAME( + "extensions:\n" + "- version: 0", + "ERROR: :2:3: Missing required attribute(s): name\n" + + " | - version: 0\n" + + " | ..^"), + ; + + private final String yamlConfig; + private final String expectedErrorMessage; + + ConfigParseErrorTestcase(String yamlConfig, String expectedErrorMessage) { + this.yamlConfig = yamlConfig; + this.expectedErrorMessage = expectedErrorMessage; + } + } +} From 99b52f5145b761e63a4b387e616eda0ffc6cf34f Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Tue, 2 Jul 2024 10:38:43 -0700 Subject: [PATCH 13/33] Create a separate interface for validating required fields in a builder PiperOrigin-RevId: 648767398 --- .../src/main/java/dev/cel/policy/BUILD.bazel | 13 +++++ .../java/dev/cel/policy/CelPolicyConfig.java | 56 ++++++++----------- .../dev/cel/policy/RequiredFieldsChecker.java | 49 ++++++++++++++++ 3 files changed, 85 insertions(+), 33 deletions(-) create mode 100644 policy/src/main/java/dev/cel/policy/RequiredFieldsChecker.java diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index 5fc24e33d..85b0f4d38 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -38,6 +38,7 @@ java_library( tags = [ ], deps = [ + ":required_fields_checker", ":source", "//:auto_value", "@maven//:com_google_errorprone_error_prone_annotations", @@ -89,6 +90,18 @@ java_library( ], ) +java_library( + name = "required_fields_checker", + srcs = [ + "RequiredFieldsChecker.java", + ], + visibility = ["//visibility:private"], + deps = [ + "//:auto_value", + "@maven//:com_google_guava_guava", + ], +) + java_library( name = "parser_context", srcs = [ diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java b/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java index 4d15c4d8b..c9b63bf3b 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java @@ -15,15 +15,12 @@ package dev.cel.policy; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.collect.ImmutableList.toImmutableList; -import static java.util.Arrays.stream; import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CheckReturnValue; -import java.util.AbstractMap; import java.util.Arrays; import java.util.Optional; @@ -108,7 +105,7 @@ public abstract static class VariableDecl { /** Builder for {@link VariableDecl}. */ @AutoValue.Builder - public abstract static class Builder { + public abstract static class Builder implements RequiredFieldsChecker { abstract Optional name(); @@ -118,10 +115,10 @@ public abstract static class Builder { public abstract Builder setType(TypeDecl typeDecl); - ImmutableList getMissingRequiredFieldNames() { - return getMissingRequiredFields( - new AbstractMap.SimpleEntry<>("name", name()), - new AbstractMap.SimpleEntry<>("type", type())); + @Override + public ImmutableList requiredFields() { + return ImmutableList.of( + RequiredField.of("name", this::name), RequiredField.of("type", this::type)); } /** Builds a new instance of {@link VariableDecl}. */ @@ -148,7 +145,7 @@ public abstract static class FunctionDecl { /** Builder for {@link FunctionDecl}. */ @AutoValue.Builder - public abstract static class Builder { + public abstract static class Builder implements RequiredFieldsChecker { abstract Optional name(); @@ -158,10 +155,10 @@ public abstract static class Builder { public abstract Builder setOverloads(ImmutableSet overloads); - ImmutableList getMissingRequiredFieldNames() { - return getMissingRequiredFields( - new AbstractMap.SimpleEntry<>("name", name()), - new AbstractMap.SimpleEntry<>("overloads", overloads())); + @Override + public ImmutableList requiredFields() { + return ImmutableList.of( + RequiredField.of("name", this::name), RequiredField.of("overloads", this::overloads)); } /** Builds a new instance of {@link FunctionDecl}. */ @@ -200,7 +197,7 @@ public abstract static class OverloadDecl { /** Builder for {@link OverloadDecl}. */ @AutoValue.Builder - public abstract static class Builder { + public abstract static class Builder implements RequiredFieldsChecker { abstract Optional id(); @@ -228,10 +225,10 @@ public Builder addArguments(TypeDecl... args) { public abstract Builder setReturnType(TypeDecl returnType); - ImmutableList getMissingRequiredFieldNames() { - return getMissingRequiredFields( - new AbstractMap.SimpleEntry<>("id", id()), - new AbstractMap.SimpleEntry<>("return", returnType())); + @Override + public ImmutableList requiredFields() { + return ImmutableList.of( + RequiredField.of("id", this::id), RequiredField.of("return", this::returnType)); } /** Builds a new instance of {@link OverloadDecl}. */ @@ -259,7 +256,7 @@ public abstract static class TypeDecl { /** Builder for {@link TypeDecl}. */ @AutoValue.Builder - public abstract static class Builder { + public abstract static class Builder implements RequiredFieldsChecker { abstract Optional name(); @@ -283,8 +280,9 @@ public Builder addParams(Iterable params) { public abstract Builder setIsTypeParam(boolean isTypeParam); - ImmutableList getMissingRequiredFieldNames() { - return getMissingRequiredFields(new AbstractMap.SimpleEntry<>("type_name", name())); + @Override + public ImmutableList requiredFields() { + return ImmutableList.of(RequiredField.of("type_name", this::name)); } @CheckReturnValue @@ -319,7 +317,7 @@ public abstract static class ExtensionConfig { /** Builder for {@link ExtensionConfig}. */ @AutoValue.Builder - public abstract static class Builder { + public abstract static class Builder implements RequiredFieldsChecker { abstract Optional name(); @@ -329,8 +327,9 @@ public abstract static class Builder { public abstract Builder setVersion(Integer version); - ImmutableList getMissingRequiredFieldNames() { - return getMissingRequiredFields(new AbstractMap.SimpleEntry<>("name", name())); + @Override + public ImmutableList requiredFields() { + return ImmutableList.of(RequiredField.of("name", this::name)); } /** Builds a new instance of {@link ExtensionConfig}. */ @@ -352,13 +351,4 @@ public static ExtensionConfig of(String name, int version) { return newBuilder().setName(name).setVersion(version).build(); } } - - @SafeVarargs - private static ImmutableList getMissingRequiredFields( - AbstractMap.SimpleEntry>... requiredFields) { - return stream(requiredFields) - .filter(entry -> !entry.getValue().isPresent()) - .map(AbstractMap.SimpleEntry::getKey) - .collect(toImmutableList()); - } } diff --git a/policy/src/main/java/dev/cel/policy/RequiredFieldsChecker.java b/policy/src/main/java/dev/cel/policy/RequiredFieldsChecker.java new file mode 100644 index 000000000..e46b2822a --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/RequiredFieldsChecker.java @@ -0,0 +1,49 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import java.util.Optional; +import java.util.function.Supplier; + +/** + * Interface to be implemented on a builder that can be used to verify all required fields being + * set. + */ +interface RequiredFieldsChecker { + + ImmutableList requiredFields(); + + default ImmutableList getMissingRequiredFieldNames() { + return requiredFields().stream() + .filter(entry -> !entry.fieldValue().get().isPresent()) + .map(RequiredField::displayName) + .collect(toImmutableList()); + } + + @AutoValue + abstract class RequiredField { + abstract String displayName(); + + abstract Supplier> fieldValue(); + + static RequiredField of(String displayName, Supplier> fieldValue) { + return new AutoValue_RequiredFieldsChecker_RequiredField(displayName, fieldValue); + } + } +} From e0f0225988fadb99962d679de58db9826f5c9d25 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Tue, 2 Jul 2024 11:32:29 -0700 Subject: [PATCH 14/33] Add the capability to extend CEL environment from parsed config PiperOrigin-RevId: 648788074 --- .../src/main/java/dev/cel/bundle/CelImpl.java | 5 + .../main/java/dev/cel/checker/CelChecker.java | 4 + .../dev/cel/checker/CelCheckerLegacyImpl.java | 9 + .../java/dev/cel/common/types/SimpleType.java | 20 ++ .../dev/cel/compiler/CelCompilerImpl.java | 5 + .../src/main/java/dev/cel/policy/BUILD.bazel | 8 + .../java/dev/cel/policy/CelPolicyConfig.java | 175 +++++++++++++++++- .../src/test/java/dev/cel/policy/BUILD.bazel | 3 + .../policy/CelPolicyYamlConfigParserTest.java | 97 ++++++++++ 9 files changed, 317 insertions(+), 9 deletions(-) diff --git a/bundle/src/main/java/dev/cel/bundle/CelImpl.java b/bundle/src/main/java/dev/cel/bundle/CelImpl.java index ad062d113..317eeb49c 100644 --- a/bundle/src/main/java/dev/cel/bundle/CelImpl.java +++ b/bundle/src/main/java/dev/cel/bundle/CelImpl.java @@ -87,6 +87,11 @@ public CelValidationResult check(CelAbstractSyntaxTree ast) { return compiler.get().check(ast); } + @Override + public CelTypeProvider getTypeProvider() { + return compiler.get().getTypeProvider(); + } + @Override public CelRuntime.Program createProgram(CelAbstractSyntaxTree ast) throws CelEvaluationException { return runtime.get().createProgram(ast); diff --git a/checker/src/main/java/dev/cel/checker/CelChecker.java b/checker/src/main/java/dev/cel/checker/CelChecker.java index 4022212cb..077850be1 100644 --- a/checker/src/main/java/dev/cel/checker/CelChecker.java +++ b/checker/src/main/java/dev/cel/checker/CelChecker.java @@ -17,6 +17,7 @@ import com.google.errorprone.annotations.Immutable; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelValidationResult; +import dev.cel.common.types.CelTypeProvider; /** Public interface for type-checking parsed CEL expressions. */ @Immutable @@ -29,5 +30,8 @@ public interface CelChecker { */ CelValidationResult check(CelAbstractSyntaxTree ast); + /** Returns the underlying type provider. */ + CelTypeProvider getTypeProvider(); + CelCheckerBuilder toCheckerBuilder(); } diff --git a/checker/src/main/java/dev/cel/checker/CelCheckerLegacyImpl.java b/checker/src/main/java/dev/cel/checker/CelCheckerLegacyImpl.java index 950c919e3..a177b259d 100644 --- a/checker/src/main/java/dev/cel/checker/CelCheckerLegacyImpl.java +++ b/checker/src/main/java/dev/cel/checker/CelCheckerLegacyImpl.java @@ -75,6 +75,7 @@ public final class CelCheckerLegacyImpl implements CelChecker, EnvVisitable { @SuppressWarnings("Immutable") private final TypeProvider typeProvider; + private final CelTypeProvider celTypeProvider; private final boolean standardEnvironmentEnabled; // Builder is mutable by design. APIs must make defensive copies in and out of this class. @@ -98,6 +99,11 @@ public CelValidationResult check(CelAbstractSyntaxTree ast) { return new CelValidationResult(checkedAst, ImmutableList.of()); } + @Override + public CelTypeProvider getTypeProvider() { + return this.celTypeProvider; + } + @Override public CelCheckerBuilder toCheckerBuilder() { return new Builder(checkerBuilder); @@ -422,6 +428,7 @@ public CelCheckerLegacyImpl build() { functionDeclarations.build(), Optional.fromNullable(expectedResultType), legacyProvider, + messageTypeProvider, standardEnvironmentEnabled, this); } @@ -469,6 +476,7 @@ private CelCheckerLegacyImpl( ImmutableSet functionDeclarations, Optional expectedResultType, TypeProvider typeProvider, + CelTypeProvider celTypeProvider, boolean standardEnvironmentEnabled, Builder checkerBuilder) { this.celOptions = celOptions; @@ -477,6 +485,7 @@ private CelCheckerLegacyImpl( this.functionDeclarations = functionDeclarations; this.expectedResultType = expectedResultType; this.typeProvider = typeProvider; + this.celTypeProvider = celTypeProvider; this.standardEnvironmentEnabled = standardEnvironmentEnabled; this.checkerBuilder = new Builder(checkerBuilder); } diff --git a/common/src/main/java/dev/cel/common/types/SimpleType.java b/common/src/main/java/dev/cel/common/types/SimpleType.java index 378c669f1..cbf02e9c6 100644 --- a/common/src/main/java/dev/cel/common/types/SimpleType.java +++ b/common/src/main/java/dev/cel/common/types/SimpleType.java @@ -15,8 +15,10 @@ package dev.cel.common.types; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CheckReturnValue; import com.google.errorprone.annotations.Immutable; +import java.util.Optional; /** Simple types represent scalar, dynamic, and error values. */ @AutoValue @@ -42,6 +44,19 @@ public abstract class SimpleType extends CelType { public static final CelType TIMESTAMP = create(CelKind.TIMESTAMP, "google.protobuf.Timestamp"); public static final CelType UINT = create(CelKind.UINT, "uint"); + private static final ImmutableMap TYPE_MAP = + ImmutableMap.of( + DYN.name(), DYN, + BOOL.name(), BOOL, + BYTES.name(), BYTES, + DOUBLE.name(), DOUBLE, + DURATION.name(), DURATION, + INT.name(), INT, + NULL_TYPE.name(), NULL_TYPE, + STRING.name(), STRING, + TIMESTAMP.name(), TIMESTAMP, + UINT.name(), UINT); + @Override public abstract CelKind kind(); @@ -56,6 +71,11 @@ public boolean isAssignableFrom(CelType other) { || (other instanceof NullableType && other.isAssignableFrom(this)); } + /** Returns a matching SimpleType by its name if one exists. */ + public static Optional findByName(String typeName) { + return Optional.ofNullable(TYPE_MAP.get(typeName)); + } + private static CelType create(CelKind kind, String name) { return new AutoValue_SimpleType(kind, name); } diff --git a/compiler/src/main/java/dev/cel/compiler/CelCompilerImpl.java b/compiler/src/main/java/dev/cel/compiler/CelCompilerImpl.java index ac81f5d0c..13da89f2f 100644 --- a/compiler/src/main/java/dev/cel/compiler/CelCompilerImpl.java +++ b/compiler/src/main/java/dev/cel/compiler/CelCompilerImpl.java @@ -74,6 +74,11 @@ public CelValidationResult check(CelAbstractSyntaxTree ast) { return checker.check(ast); } + @Override + public CelTypeProvider getTypeProvider() { + return checker.getTypeProvider(); + } + @Override public void accept(EnvVisitor envVisitor) { if (checker instanceof EnvVisitable) { diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index 85b0f4d38..03fc041b6 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -40,7 +40,15 @@ java_library( deps = [ ":required_fields_checker", ":source", + ":validation_exception", "//:auto_value", + "//bundle:cel", + "//common:compiler_common", + "//common:options", + "//common/types", + "//common/types:type_providers", + "//extensions", + "//extensions:optional_library", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", ], diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java b/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java index c9b63bf3b..13f2bad66 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java @@ -15,12 +15,29 @@ package dev.cel.policy; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.base.Preconditions.checkState; +import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.CheckReturnValue; +import dev.cel.bundle.Cel; +import dev.cel.bundle.CelBuilder; +import dev.cel.common.CelFunctionDecl; +import dev.cel.common.CelOptions; +import dev.cel.common.CelOverloadDecl; +import dev.cel.common.CelVarDecl; +import dev.cel.common.types.CelType; +import dev.cel.common.types.CelTypeProvider; +import dev.cel.common.types.ListType; +import dev.cel.common.types.MapType; +import dev.cel.common.types.OptionalType; +import dev.cel.common.types.SimpleType; +import dev.cel.common.types.TypeParamType; +import dev.cel.extensions.CelExtensions; +import dev.cel.extensions.CelOptionalLibrary; import java.util.Arrays; import java.util.Optional; @@ -93,6 +110,66 @@ public static Builder newBuilder() { .setFunctions(ImmutableSet.of()); } + /** Extends the provided {@code cel} environment with this configuration. */ + public Cel extend(Cel cel, CelOptions celOptions) throws CelPolicyValidationException { + try { + CelTypeProvider celTypeProvider = cel.getTypeProvider(); + CelBuilder celBuilder = + cel.toCelBuilder() + .setTypeProvider(celTypeProvider) + .setContainer(container()) + .addVarDeclarations( + variables().stream() + .map(v -> v.toCelVarDecl(celTypeProvider)) + .collect(toImmutableList())) + .addFunctionDeclarations( + functions().stream() + .map(f -> f.toCelFunctionDecl(celTypeProvider)) + .collect(toImmutableList())); + + addAllExtensions(celBuilder, celOptions); + + return celBuilder.build(); + } catch (RuntimeException e) { + throw new CelPolicyValidationException(e.getMessage(), e); + } + } + + private void addAllExtensions(CelBuilder celBuilder, CelOptions celOptions) { + for (ExtensionConfig extensionConfig : extensions()) { + switch (extensionConfig.name()) { + case "bindings": + celBuilder.addCompilerLibraries(CelExtensions.bindings()); + break; + case "encoders": + celBuilder.addCompilerLibraries(CelExtensions.encoders()); + celBuilder.addRuntimeLibraries(CelExtensions.encoders()); + break; + case "math": + celBuilder.addCompilerLibraries(CelExtensions.math(celOptions)); + celBuilder.addRuntimeLibraries(CelExtensions.math(celOptions)); + break; + case "optional": + celBuilder.addCompilerLibraries(CelOptionalLibrary.INSTANCE); + celBuilder.addRuntimeLibraries(CelOptionalLibrary.INSTANCE); + break; + case "protos": + celBuilder.addCompilerLibraries(CelExtensions.protos()); + break; + case "strings": + celBuilder.addCompilerLibraries(CelExtensions.strings()); + celBuilder.addRuntimeLibraries(CelExtensions.strings()); + break; + case "sets": + celBuilder.addCompilerLibraries(CelExtensions.sets()); + celBuilder.addRuntimeLibraries(CelExtensions.sets()); + break; + default: + throw new IllegalArgumentException("Unrecognized extension: " + extensionConfig.name()); + } + } + } + /** Represents a policy variable declaration. */ @AutoValue public abstract static class VariableDecl { @@ -107,9 +184,9 @@ public abstract static class VariableDecl { @AutoValue.Builder public abstract static class Builder implements RequiredFieldsChecker { - abstract Optional name(); + public abstract Optional name(); - abstract Optional type(); + public abstract Optional type(); public abstract Builder setName(String name); @@ -133,6 +210,11 @@ public static Builder newBuilder() { public static VariableDecl create(String name, TypeDecl type) { return newBuilder().setName(name).setType(type).build(); } + + /** Converts this policy variable declaration into a {@link CelVarDecl}. */ + public CelVarDecl toCelVarDecl(CelTypeProvider celTypeProvider) { + return CelVarDecl.newVarDeclaration(name(), type().toCelType(celTypeProvider)); + } } /** Represents a policy function declaration. */ @@ -147,9 +229,9 @@ public abstract static class FunctionDecl { @AutoValue.Builder public abstract static class Builder implements RequiredFieldsChecker { - abstract Optional name(); + public abstract Optional name(); - abstract Optional> overloads(); + public abstract Optional> overloads(); public abstract Builder setName(String name); @@ -174,6 +256,15 @@ public static Builder newBuilder() { public static FunctionDecl create(String name, ImmutableSet overloads) { return newBuilder().setName(name).setOverloads(overloads).build(); } + + /** Converts this policy function declaration into a {@link CelFunctionDecl}. */ + public CelFunctionDecl toCelFunctionDecl(CelTypeProvider celTypeProvider) { + return CelFunctionDecl.newFunctionDeclaration( + name(), + overloads().stream() + .map(o -> o.toCelOverloadDecl(celTypeProvider)) + .collect(toImmutableList())); + } } /** Represents an overload declaraion on a policy function. */ @@ -199,9 +290,9 @@ public abstract static class OverloadDecl { @AutoValue.Builder public abstract static class Builder implements RequiredFieldsChecker { - abstract Optional id(); + public abstract Optional id(); - abstract Optional returnType(); + public abstract Optional returnType(); public abstract Builder setId(String overloadId); @@ -240,6 +331,28 @@ public ImmutableList requiredFields() { public static Builder newBuilder() { return new AutoValue_CelPolicyConfig_OverloadDecl.Builder().setArguments(ImmutableList.of()); } + + /** Converts this policy function overload into a {@link CelOverloadDecl}. */ + public CelOverloadDecl toCelOverloadDecl(CelTypeProvider celTypeProvider) { + CelOverloadDecl.Builder builder = + CelOverloadDecl.newBuilder() + .setIsInstanceFunction(false) + .setOverloadId(id()) + .setResultType(returnType().toCelType(celTypeProvider)); + + target() + .ifPresent( + t -> + builder + .setIsInstanceFunction(true) + .addParameterTypes(t.toCelType(celTypeProvider))); + + for (TypeDecl type : arguments()) { + builder.addParameterTypes(type.toCelType(celTypeProvider)); + } + + return builder.build(); + } } /** @@ -258,7 +371,7 @@ public abstract static class TypeDecl { @AutoValue.Builder public abstract static class Builder implements RequiredFieldsChecker { - abstract Optional name(); + public abstract Optional name(); public abstract Builder setName(String name); @@ -297,6 +410,50 @@ public static TypeDecl create(String name) { public static Builder newBuilder() { return new AutoValue_CelPolicyConfig_TypeDecl.Builder().setIsTypeParam(false); } + + /** Converts this type declaration into a {@link CelType}. */ + public CelType toCelType(CelTypeProvider celTypeProvider) { + switch (name()) { + case "list": + if (params().size() != 1) { + throw new IllegalArgumentException( + "List type has unexpected param count: " + params().size()); + } + + CelType elementType = params().get(0).toCelType(celTypeProvider); + return ListType.create(elementType); + case "map": + if (params().size() != 2) { + throw new IllegalArgumentException( + "Map type has unexpected param count: " + params().size()); + } + + CelType keyType = params().get(0).toCelType(celTypeProvider); + CelType valueType = params().get(1).toCelType(celTypeProvider); + return MapType.create(keyType, valueType); + default: + if (isTypeParam()) { + return TypeParamType.create(name()); + } + + CelType simpleType = SimpleType.findByName(name()).orElse(null); + if (simpleType != null) { + return simpleType; + } + + if (name().equals(OptionalType.NAME)) { + checkState( + params().size() == 1, + "Optional type must have exactly 1 parameter. Found %s", + params().size()); + return OptionalType.create(params().get(0).toCelType(celTypeProvider)); + } + + return celTypeProvider + .findType(name()) + .orElseThrow(() -> new IllegalArgumentException("Undefined type name: " + name())); + } + } } /** @@ -319,9 +476,9 @@ public abstract static class ExtensionConfig { @AutoValue.Builder public abstract static class Builder implements RequiredFieldsChecker { - abstract Optional name(); + public abstract Optional name(); - abstract Optional version(); + public abstract Optional version(); public abstract Builder setName(String name); diff --git a/policy/src/test/java/dev/cel/policy/BUILD.bazel b/policy/src/test/java/dev/cel/policy/BUILD.bazel index 951b45f6a..b4844833a 100644 --- a/policy/src/test/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/test/java/dev/cel/policy/BUILD.bazel @@ -8,12 +8,15 @@ java_library( srcs = glob(["*.java"]), deps = [ "//:java_truth", + "//bundle:cel", + "//common:options", "//common/internal", "//policy:config", "//policy:config_parser", "//policy:source", "//policy:validation_exception", "//policy:yaml_config_parser", + "@maven//:com_google_api_grpc_proto_google_common_protos", "@maven//:com_google_guava_guava", "@maven//:com_google_testparameterinjector_test_parameter_injector", "@maven//:junit_junit", diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java index 020d9f689..235126270 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java @@ -18,8 +18,12 @@ import static org.junit.Assert.assertThrows; import com.google.common.collect.ImmutableSet; +import com.google.rpc.context.AttributeContext; import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.bundle.Cel; +import dev.cel.bundle.CelFactory; +import dev.cel.common.CelOptions; import dev.cel.policy.CelPolicyConfig.ExtensionConfig; import dev.cel.policy.CelPolicyConfig.FunctionDecl; import dev.cel.policy.CelPolicyConfig.OverloadDecl; @@ -31,6 +35,11 @@ @RunWith(TestParameterInjector.class) public final class CelPolicyYamlConfigParserTest { + private static final Cel CEL_WITH_MESSAGE_TYPES = + CelFactory.standardCelBuilder() + .addMessageTypes(AttributeContext.Request.getDescriptor()) + .build(); + private static final CelPolicyConfigParser POLICY_CONFIG_PARSER = CelPolicyYamlConfigParser.newInstance(); @@ -79,6 +88,7 @@ public void config_setExtensions() throws Exception { ExtensionConfig.of("sets"), ExtensionConfig.of("strings", 1))) .build()); + assertThat(policyConfig.extend(CEL_WITH_MESSAGE_TYPES, CelOptions.DEFAULT)).isNotNull(); } @Test @@ -172,6 +182,7 @@ public void config_setFunctions() throws Exception { .build()) .build())))) .build()); + assertThat(policyConfig.extend(CEL_WITH_MESSAGE_TYPES, CelOptions.DEFAULT)).isNotNull(); } @Test @@ -227,6 +238,7 @@ public void config_setMapVariable() throws Exception { .addParams(TypeDecl.create("string"), TypeDecl.create("dyn")) .build()))) .build()); + assertThat(policyConfig.extend(CEL_WITH_MESSAGE_TYPES, CelOptions.DEFAULT)).isNotNull(); } @Test @@ -249,6 +261,7 @@ public void config_setMessageVariable() throws Exception { "request", TypeDecl.create("google.rpc.context.AttributeContext.Request")))) .build()); + assertThat(policyConfig.extend(CEL_WITH_MESSAGE_TYPES, CelOptions.DEFAULT)).isNotNull(); } @Test @@ -260,6 +273,18 @@ public void config_parseErrors(@TestParameter ConfigParseErrorTestcase testCase) assertThat(e).hasMessageThat().isEqualTo(testCase.expectedErrorMessage); } + @Test + public void config_extendErrors(@TestParameter ConfigExtendErrorTestCase testCase) + throws Exception { + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(testCase.yamlConfig); + + CelPolicyValidationException e = + assertThrows( + CelPolicyValidationException.class, + () -> policyConfig.extend(CEL_WITH_MESSAGE_TYPES, CelOptions.DEFAULT)); + assertThat(e).hasMessageThat().isEqualTo(testCase.expectedErrorMessage); + } + // Note: dangling comments in expressions below is to retain the newlines by preventing auto // formatter from compressing them in a single line. private enum ConfigParseErrorTestcase { @@ -440,4 +465,76 @@ private enum ConfigParseErrorTestcase { this.expectedErrorMessage = expectedErrorMessage; } } + + private enum ConfigExtendErrorTestCase { + BAD_EXTENSION("extensions:\n" + " - name: 'bad_name'", "Unrecognized extension: bad_name"), + BAD_TYPE( + "variables:\n" + "- name: 'bad_type'\n" + " type:\n" + " type_name: 'strings'", + "Undefined type name: strings"), + BAD_LIST( + "variables:\n" + " - name: 'bad_list'\n" + " type:\n" + " type_name: 'list'", + "List type has unexpected param count: 0"), + BAD_MAP( + "variables:\n" + + " - name: 'bad_map'\n" + + " type:\n" + + " type_name: 'map'\n" + + " params:\n" + + " - type_name: 'string'", + "Map type has unexpected param count: 1"), + BAD_LIST_TYPE_PARAM( + "variables:\n" + + " - name: 'bad_list_type_param'\n" + + " type:\n" + + " type_name: 'list'\n" + + " params:\n" + + " - type_name: 'number'", + "Undefined type name: number"), + BAD_MAP_TYPE_PARAM( + "variables:\n" + + " - name: 'bad_map_type_param'\n" + + " type:\n" + + " type_name: 'map'\n" + + " params:\n" + + " - type_name: 'string'\n" + + " - type_name: 'optional'", + "Undefined type name: optional"), + BAD_RETURN( + "functions:\n" + + " - name: 'bad_return'\n" + + " overloads:\n" + + " - id: 'zero_arity'\n" + + " return:\n" + + " type_name: 'mystery'", + "Undefined type name: mystery"), + BAD_OVERLOAD_TARGET( + "functions:\n" + + " - name: 'bad_target'\n" + + " overloads:\n" + + " - id: 'unary_member'\n" + + " target:\n" + + " type_name: 'unknown'\n" + + " return:\n" + + " type_name: 'null_type'", + "Undefined type name: unknown"), + BAD_OVERLOAD_ARG( + "functions:\n" + + " - name: 'bad_arg'\n" + + " overloads:\n" + + " - id: 'unary_global'\n" + + " args:\n" + + " - type_name: 'unknown'\n" + + " return:\n" + + " type_name: 'null_type'", + "Undefined type name: unknown"), + ; + + private final String yamlConfig; + private final String expectedErrorMessage; + + ConfigExtendErrorTestCase(String yamlConfig, String expectedErrorMessage) { + this.yamlConfig = yamlConfig; + this.expectedErrorMessage = expectedErrorMessage; + } + } } From b9fe54ded9d718019824852e77d0796884c716f5 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Tue, 2 Jul 2024 13:20:26 -0700 Subject: [PATCH 15/33] Add YAML policy test cases PiperOrigin-RevId: 648822357 --- policy/src/test/resources/BUILD.bazel | 14 +++ policy/src/test/resources/errors/config.yaml | 52 ++++++++ policy/src/test/resources/errors/policy.yaml | 34 +++++ .../test/resources/nested_rule/config.yaml | 22 ++++ .../test/resources/nested_rule/policy.yaml | 38 ++++++ .../src/test/resources/nested_rule/tests.yaml | 38 ++++++ .../resources/required_labels/config.yaml | 32 +++++ .../resources/required_labels/policy.yaml | 32 +++++ .../test/resources/required_labels/tests.yaml | 79 ++++++++++++ .../restricted_destinations/config.yaml | 52 ++++++++ .../restricted_destinations/policy.yaml | 42 +++++++ .../restricted_destinations/tests.yaml | 118 ++++++++++++++++++ 12 files changed, 553 insertions(+) create mode 100644 policy/src/test/resources/BUILD.bazel create mode 100644 policy/src/test/resources/errors/config.yaml create mode 100644 policy/src/test/resources/errors/policy.yaml create mode 100644 policy/src/test/resources/nested_rule/config.yaml create mode 100644 policy/src/test/resources/nested_rule/policy.yaml create mode 100644 policy/src/test/resources/nested_rule/tests.yaml create mode 100644 policy/src/test/resources/required_labels/config.yaml create mode 100644 policy/src/test/resources/required_labels/policy.yaml create mode 100644 policy/src/test/resources/required_labels/tests.yaml create mode 100644 policy/src/test/resources/restricted_destinations/config.yaml create mode 100644 policy/src/test/resources/restricted_destinations/policy.yaml create mode 100644 policy/src/test/resources/restricted_destinations/tests.yaml diff --git a/policy/src/test/resources/BUILD.bazel b/policy/src/test/resources/BUILD.bazel new file mode 100644 index 000000000..71285b61a --- /dev/null +++ b/policy/src/test/resources/BUILD.bazel @@ -0,0 +1,14 @@ +package( + default_applicable_licenses = [ + "//:license", + ], + default_testonly = True, + default_visibility = [ + "//policy:__subpackages__", + ], +) + +filegroup( + name = "policy_yaml_files", + srcs = glob(["**/*.yaml"]), +) diff --git a/policy/src/test/resources/errors/config.yaml b/policy/src/test/resources/errors/config.yaml new file mode 100644 index 000000000..b9c8f9750 --- /dev/null +++ b/policy/src/test/resources/errors/config.yaml @@ -0,0 +1,52 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "labels" +extensions: + - name: "sets" +variables: + - name: "destination.ip" + type: + type_name: "string" + - name: "origin.ip" + type: + type_name: "string" + - name: "spec.restricted_destinations" + type: + type_name: "list" + params: + - type_name: "string" + - name: "spec.origin" + type: + type_name: "string" + - name: "request" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "dyn" + - name: "resource" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "dyn" +functions: + - name: "locationCode" + overloads: + - id: "locationCode_string" + args: + - type_name: "string" + return: + type_name: "string" diff --git a/policy/src/test/resources/errors/policy.yaml b/policy/src/test/resources/errors/policy.yaml new file mode 100644 index 000000000..338ba3995 --- /dev/null +++ b/policy/src/test/resources/errors/policy.yaml @@ -0,0 +1,34 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "errors" +rule: + variables: + - name: want + expression: spec.labels + - name: missing + expression: variables.want.filter(l, !(lin resource.labels)) + - name: bad_data + expression: "{1:305 2:569}" + - name: invalid + expression: > + resource.labels.filter(l, + l in variables.want && variables.want[l] != resource.labels[l]) + match: + - condition: variables.missing.size() > 0 + output: | + "missing one or more required labels: %s".format(variables.missing]) + - condition: variables.invalid.size() > 0 + output: | + "invalid values provided on one or more labels: %s".format([variables.invalid]) diff --git a/policy/src/test/resources/nested_rule/config.yaml b/policy/src/test/resources/nested_rule/config.yaml new file mode 100644 index 000000000..bfd94b33c --- /dev/null +++ b/policy/src/test/resources/nested_rule/config.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "nested_rule" +variables: + - name: "resource" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "dyn" diff --git a/policy/src/test/resources/nested_rule/policy.yaml b/policy/src/test/resources/nested_rule/policy.yaml new file mode 100644 index 000000000..bbbfe0fc1 --- /dev/null +++ b/policy/src/test/resources/nested_rule/policy.yaml @@ -0,0 +1,38 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: nested_rule +rule: + variables: + - name: "permitted_regions" + expression: "['us', 'uk', 'es']" + match: + - rule: + id: "banned regions" + description: > + determine whether the resource origin is in the banned + list. If the region is also in the permitted list, the + ban has no effect. + variables: + - name: "banned_regions" + expression: "{'us': false, 'ru': false, 'ir': false}" + match: + - condition: | + resource.origin in variables.banned_regions && + !(resource.origin in variables.permitted_regions) + output: "{'banned': true}" + - condition: resource.origin in variables.permitted_regions + output: "{'banned': false}" + - output: "{'banned': true}" + diff --git a/policy/src/test/resources/nested_rule/tests.yaml b/policy/src/test/resources/nested_rule/tests.yaml new file mode 100644 index 000000000..a9807c376 --- /dev/null +++ b/policy/src/test/resources/nested_rule/tests.yaml @@ -0,0 +1,38 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description: Nested rule conformance tests +section: + - name: "banned" + tests: + - name: "restricted_origin" + input: + resource: + value: + origin: "ir" + output: "{'banned': true}" + - name: "by_default" + input: + resource: + value: + origin: "de" + output: "{'banned': true}" + - name: "permitted" + tests: + - name: "valid_origin" + input: + resource: + value: + origin: "uk" + output: "{'banned': false}" diff --git a/policy/src/test/resources/required_labels/config.yaml b/policy/src/test/resources/required_labels/config.yaml new file mode 100644 index 000000000..14311d763 --- /dev/null +++ b/policy/src/test/resources/required_labels/config.yaml @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "labels" +extensions: + - name: "bindings" + - name: "strings" + version: 2 +variables: + - name: "spec" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "dyn" + - name: "resource" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "dyn" diff --git a/policy/src/test/resources/required_labels/policy.yaml b/policy/src/test/resources/required_labels/policy.yaml new file mode 100644 index 000000000..aca75290f --- /dev/null +++ b/policy/src/test/resources/required_labels/policy.yaml @@ -0,0 +1,32 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "required_labels" +rule: + variables: + - name: want + expression: spec.labels + - name: missing + expression: variables.want.filter(l, !(l in resource.labels)) + - name: invalid + expression: > + resource.labels.filter(l, + l in variables.want && variables.want[l] != resource.labels[l]) + match: + - condition: variables.missing.size() > 0 + output: | + "missing one or more required labels: [\"" + variables.missing.join(',') + "\"]" + - condition: variables.invalid.size() > 0 + output: | + "invalid values provided on one or more labels: [\"" + variables.invalid.join(',') + "\"]" diff --git a/policy/src/test/resources/required_labels/tests.yaml b/policy/src/test/resources/required_labels/tests.yaml new file mode 100644 index 000000000..67681ef46 --- /dev/null +++ b/policy/src/test/resources/required_labels/tests.yaml @@ -0,0 +1,79 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description: "Required labels conformance tests" +section: + - name: "valid" + tests: + - name: "matching" + input: + spec: + value: + labels: + env: prod + experiment: "group b" + resource: + value: + labels: + env: prod + experiment: "group b" + release: "v0.1.0" + output: "optional.none()" + - name: "missing" + tests: + - name: "env" + input: + spec: + value: + labels: + env: prod + experiment: "group b" + resource: + value: + labels: + experiment: "group b" + release: "v0.1.0" + output: > + "missing one or more required labels: [\"env\"]" + - name: "experiment" + input: + spec: + value: + labels: + env: prod + experiment: "group b" + resource: + value: + labels: + env: staging + release: "v0.1.0" + output: > + "missing one or more required labels: [\"experiment\"]" + - name: "invalid" + tests: + - name: "env" + input: + spec: + value: + labels: + env: prod + experiment: "group b" + resource: + value: + labels: + env: staging + experiment: "group b" + release: "v0.1.0" + output: > + "invalid values provided on one or more labels: [\"env\"]" diff --git a/policy/src/test/resources/restricted_destinations/config.yaml b/policy/src/test/resources/restricted_destinations/config.yaml new file mode 100644 index 000000000..b9c8f9750 --- /dev/null +++ b/policy/src/test/resources/restricted_destinations/config.yaml @@ -0,0 +1,52 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "labels" +extensions: + - name: "sets" +variables: + - name: "destination.ip" + type: + type_name: "string" + - name: "origin.ip" + type: + type_name: "string" + - name: "spec.restricted_destinations" + type: + type_name: "list" + params: + - type_name: "string" + - name: "spec.origin" + type: + type_name: "string" + - name: "request" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "dyn" + - name: "resource" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "dyn" +functions: + - name: "locationCode" + overloads: + - id: "locationCode_string" + args: + - type_name: "string" + return: + type_name: "string" diff --git a/policy/src/test/resources/restricted_destinations/policy.yaml b/policy/src/test/resources/restricted_destinations/policy.yaml new file mode 100644 index 000000000..95fb454d7 --- /dev/null +++ b/policy/src/test/resources/restricted_destinations/policy.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "restricted_destinations" +rule: + variables: + - name: matches_origin_ip + expression: > + locationCode(origin.ip) == spec.origin + - name: has_nationality + expression: > + has(request.auth.claims.nationality) + - name: matches_nationality + expression: > + variables.has_nationality && request.auth.claims.nationality == spec.origin + - name: matches_dest_ip + expression: > + locationCode(destination.ip) in spec.restricted_destinations + - name: matches_dest_label + expression: > + resource.labels.location in spec.restricted_destinations + - name: matches_dest + expression: > + variables.matches_dest_ip || variables.matches_dest_label + match: + - condition: variables.matches_nationality && variables.matches_dest + output: "true" + - condition: > + !variables.has_nationality && variables.matches_origin_ip && variables.matches_dest + output: "true" + - output: "false" diff --git a/policy/src/test/resources/restricted_destinations/tests.yaml b/policy/src/test/resources/restricted_destinations/tests.yaml new file mode 100644 index 000000000..c0feeb202 --- /dev/null +++ b/policy/src/test/resources/restricted_destinations/tests.yaml @@ -0,0 +1,118 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description: Restricted destinations conformance tests. +section: + - name: "valid" + tests: + - name: "ip_allowed" + input: + "spec.origin": + value: "us" + "spec.restricted_destinations": + value: + - "cu" + - "ir" + - "kp" + - "sd" + - "sy" + "destination.ip": + value: "10.0.0.1" + "origin.ip": + value: "10.0.0.1" + request: + value: + auth: + claims: {} + resource: + value: + name: "/company/acme/secrets/doomsday-device" + labels: + location: "us" + output: "false" # false means unrestricted + - name: "nationality_allowed" + input: + "spec.origin": + value: "us" + "spec.restricted_destinations": + value: + - "cu" + - "ir" + - "kp" + - "sd" + - "sy" + "destination.ip": + value: "10.0.0.1" + request: + value: + auth: + claims: + nationality: "us" + resource: + value: + name: "/company/acme/secrets/doomsday-device" + labels: + location: "us" + output: "false" + - name: "invalid" + tests: + - name: "destination_ip_prohibited" + input: + "spec.origin": + value: "us" + "spec.restricted_destinations": + value: + - "cu" + - "ir" + - "kp" + - "sd" + - "sy" + "destination.ip": + value: "123.123.123.123" + "origin.ip": + value: "10.0.0.1" + request: + value: + auth: + claims: {} + resource: + value: + name: "/company/acme/secrets/doomsday-device" + labels: + location: "us" + output: "true" # true means restricted + - name: "resource_nationality_prohibited" + input: + "spec.origin": + value: "us" + "spec.restricted_destinations": + value: + - "cu" + - "ir" + - "kp" + - "sd" + - "sy" + "destination.ip": + value: "10.0.0.1" + request: + value: + auth: + claims: + nationality: "us" + resource: + value: + name: "/company/acme/secrets/doomsday-device" + labels: + location: "cu" + output: "true" From d63a63bb70f110f93022ee715a654365bb641077 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Tue, 2 Jul 2024 14:06:54 -0700 Subject: [PATCH 16/33] Implement YAML parser for policies PiperOrigin-RevId: 648836979 --- common/BUILD.bazel | 10 +- .../src/main/java/dev/cel/common/BUILD.bazel | 28 +- .../main/java/dev/cel/common/CelSource.java | 6 +- .../src/main/java/dev/cel/common/Source.java | 6 + policy/BUILD.bazel | 16 + .../src/main/java/dev/cel/policy/BUILD.bazel | 58 ++++ .../main/java/dev/cel/policy/CelPolicy.java | 214 +++++++++++++ .../java/dev/cel/policy/CelPolicyConfig.java | 2 +- .../java/dev/cel/policy/CelPolicyParser.java | 87 ++++++ .../cel/policy/CelPolicyParserBuilder.java | 35 +++ .../java/dev/cel/policy/CelPolicySource.java | 10 +- .../cel/policy/CelPolicyYamlConfigParser.java | 10 +- .../dev/cel/policy/CelPolicyYamlParser.java | 288 ++++++++++++++++++ .../java/dev/cel/policy/ParserContext.java | 3 + .../dev/cel/policy/YamlParserContextImpl.java | 6 + .../src/test/java/dev/cel/policy/BUILD.bazel | 6 + .../cel/policy/CelPolicyYamlParserTest.java | 202 ++++++++++++ .../java/dev/cel/policy/PolicyTestHelper.java | 105 +++++++ 18 files changed, 1082 insertions(+), 10 deletions(-) create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicy.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyParser.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyParserBuilder.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java create mode 100644 policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java create mode 100644 policy/src/test/java/dev/cel/policy/PolicyTestHelper.java diff --git a/common/BUILD.bazel b/common/BUILD.bazel index e3d61aad9..3af0c0c66 100644 --- a/common/BUILD.bazel +++ b/common/BUILD.bazel @@ -5,7 +5,10 @@ package( java_library( name = "common", - exports = ["//common/src/main/java/dev/cel/common"], + exports = [ + "//common/src/main/java/dev/cel/common", + "//common/src/main/java/dev/cel/common:source_location", # TODO: Split callers + ], ) java_library( @@ -65,3 +68,8 @@ java_library( visibility = ["//visibility:public"], exports = ["//common/src/main/java/dev/cel/common:source"], ) + +java_library( + name = "source_location", + exports = ["//common/src/main/java/dev/cel/common:source_location"], +) diff --git a/common/src/main/java/dev/cel/common/BUILD.bazel b/common/src/main/java/dev/cel/common/BUILD.bazel index 8036dafba..82e74bf1d 100644 --- a/common/src/main/java/dev/cel/common/BUILD.bazel +++ b/common/src/main/java/dev/cel/common/BUILD.bazel @@ -14,7 +14,6 @@ COMMON_SOURCES = [ "CelException.java", "CelProtoAbstractSyntaxTree.java", # TODO Split target after migrating callers "CelSource.java", - "CelSourceLocation.java", ] # keep sorted @@ -27,6 +26,12 @@ COMPILER_COMMON_SOURCES = [ "CelVarDecl.java", ] +# keep sorted +SOURCE_SOURCES = [ + "CelSourceHelper.java", + "Source.java", +] + # keep sorted PROTO_V1ALPHA1_AST_SOURCE = [ "CelProtoV1Alpha1AbstractSyntaxTree.java", @@ -40,6 +45,7 @@ java_library( deps = [ ":error_codes", ":source", + ":source_location", "//:auto_value", "//common/annotations", "//common/ast", @@ -64,6 +70,7 @@ java_library( deps = [ ":common", ":source", + ":source_location", "//:auto_value", "//common/annotations", "//common/internal:safe_string_formatter", @@ -179,11 +186,22 @@ java_library( ) java_library( - name = "source", - srcs = [ - "CelSourceHelper.java", - "Source.java", + name = "source_location", + srcs = ["CelSourceLocation.java"], + tags = [ + "alt_dep=//common:source_location", + "avoid_dep", + ], + deps = [ + "//:auto_value", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", ], +) + +java_library( + name = "source", + srcs = SOURCE_SOURCES, deps = [ "//common/annotations", "//common/internal", diff --git a/common/src/main/java/dev/cel/common/CelSource.java b/common/src/main/java/dev/cel/common/CelSource.java index b8ffe2598..574946fd3 100644 --- a/common/src/main/java/dev/cel/common/CelSource.java +++ b/common/src/main/java/dev/cel/common/CelSource.java @@ -64,6 +64,7 @@ public String getDescription() { return description; } + @Override public ImmutableMap getPositionsMap() { return positions; } @@ -170,7 +171,10 @@ public static Builder newBuilder() { } public static Builder newBuilder(String text) { - CelCodePointArray codePointArray = CelCodePointArray.fromString(text); + return newBuilder(CelCodePointArray.fromString(text)); + } + + public static Builder newBuilder(CelCodePointArray codePointArray) { return new Builder(codePointArray, codePointArray.lineOffsets()); } diff --git a/common/src/main/java/dev/cel/common/Source.java b/common/src/main/java/dev/cel/common/Source.java index 33a9927a6..2d43a9581 100644 --- a/common/src/main/java/dev/cel/common/Source.java +++ b/common/src/main/java/dev/cel/common/Source.java @@ -14,6 +14,7 @@ package dev.cel.common; +import com.google.common.collect.ImmutableMap; import dev.cel.common.annotations.Internal; import dev.cel.common.internal.CelCodePointArray; import java.util.Optional; @@ -36,6 +37,11 @@ public interface Source { */ String getDescription(); + /** + * Gets the map of each parsed node ID (ex: expression ID, policy ID) to their source positions. + */ + ImmutableMap getPositionsMap(); + /** * Get the text from the source text that corresponds to {@code line}. Snippets are split based on * the newline ('\n'). diff --git a/policy/BUILD.bazel b/policy/BUILD.bazel index 487bce068..8e65f4229 100644 --- a/policy/BUILD.bazel +++ b/policy/BUILD.bazel @@ -3,6 +3,11 @@ package( default_visibility = ["//visibility:public"], # TODO: Expose to public ) +java_library( + name = "policy", + exports = ["//policy/src/main/java/dev/cel/policy"], +) + java_library( name = "source", exports = ["//policy/src/main/java/dev/cel/policy:source"], @@ -23,6 +28,17 @@ java_library( exports = ["//policy/src/main/java/dev/cel/policy:config_parser"], ) +java_library( + name = "parser", + exports = ["//policy/src/main/java/dev/cel/policy:parser"], +) + +java_library( + name = "yaml_parser", + visibility = ["//visibility:public"], + exports = ["//policy/src/main/java/dev/cel/policy:yaml_parser"], +) + java_library( name = "yaml_config_parser", visibility = ["//visibility:public"], diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index 03fc041b6..ed152b9f2 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -5,6 +5,21 @@ package( ], ) +java_library( + name = "policy", + srcs = [ + "CelPolicy.java", + ], + deps = [ + ":required_fields_checker", + ":source", + ":value_string", + "//:auto_value", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + java_library( name = "source", srcs = [ @@ -67,6 +82,49 @@ java_library( ], ) +java_library( + name = "yaml_parser", + srcs = [ + "CelPolicyYamlParser.java", + ], + deps = [ + ":parser", + ":parser_context", + ":policy", + ":policy_common_internal", + ":policy_parser_builder", + ":source", + ":validation_exception", + ":value_string", + "//common/internal", + "@maven//:com_google_guava_guava", + "@maven//:org_yaml_snakeyaml", + ], +) + +java_library( + name = "parser", + srcs = [ + "CelPolicyParser.java", + ], + deps = [ + ":parser_context", + ":policy", + ":validation_exception", + ], +) + +java_library( + name = "policy_parser_builder", + srcs = [ + "CelPolicyParserBuilder.java", + ], + deps = [ + ":parser", + "@maven//:com_google_errorprone_error_prone_annotations", + ], +) + java_library( name = "yaml_config_parser", srcs = [ diff --git a/policy/src/main/java/dev/cel/policy/CelPolicy.java b/policy/src/main/java/dev/cel/policy/CelPolicy.java new file mode 100644 index 000000000..f64d8cc58 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicy.java @@ -0,0 +1,214 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.auto.value.AutoOneOf; +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import java.util.Arrays; +import java.util.Optional; + +/** + * Abstract representation of a policy. It declares a name, rule, and evaluation semantic for a + * given expression graph. + */ +@AutoValue +public abstract class CelPolicy { + + public abstract ValueString name(); + + public abstract Rule rule(); + + public abstract CelPolicySource policySource(); + + /** Creates a new builder to construct a {@link CelPolicy} instance. */ + public static Builder newBuilder() { + return new AutoValue_CelPolicy.Builder() + .setName(ValueString.of(0, "")) + .setRule(Rule.newBuilder().build()); + } + + /** Builder for {@link CelPolicy}. */ + @AutoValue.Builder + public abstract static class Builder { + public abstract CelPolicySource policySource(); + + public abstract Builder setName(ValueString name); + + public abstract Builder setRule(Rule rule); + + public abstract Builder setPolicySource(CelPolicySource policySource); + + public abstract CelPolicy build(); + } + + /** + * Rule declares a rule identifier, description, along with a set of variables and match + * statements. + */ + @AutoValue + public abstract static class Rule { + + public abstract Optional id(); + + public abstract Optional description(); + + public abstract ImmutableSet variables(); + + public abstract ImmutableSet matches(); + + /** Builder for {@link Rule}. */ + public static Builder newBuilder() { + return new AutoValue_CelPolicy_Rule.Builder() + .setVariables(ImmutableSet.of()) + .setMatches(ImmutableSet.of()); + } + + /** Creates a new builder to construct a {@link Rule} instance. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Rule.Builder setId(ValueString id); + + public abstract Rule.Builder setDescription(ValueString description); + + abstract ImmutableSet variables(); + + abstract ImmutableSet.Builder variablesBuilder(); + + abstract ImmutableSet matches(); + + abstract ImmutableSet.Builder matchesBuilder(); + + @CanIgnoreReturnValue + public Builder addVariables(Variable... variables) { + return addVariables(Arrays.asList(variables)); + } + + @CanIgnoreReturnValue + public Builder addVariables(Iterable variables) { + this.variablesBuilder().addAll(checkNotNull(variables)); + return this; + } + + @CanIgnoreReturnValue + public Builder addMatches(Match... matches) { + return addMatches(Arrays.asList(matches)); + } + + @CanIgnoreReturnValue + public Builder addMatches(Iterable matches) { + this.matchesBuilder().addAll(checkNotNull(matches)); + return this; + } + + abstract Rule.Builder setVariables(ImmutableSet variables); + + abstract Rule.Builder setMatches(ImmutableSet matches); + + public abstract Rule build(); + } + } + + /** + * Match declares a condition (defaults to true) as well as an output or a rule. Either the output + * or the rule field may be set, but not both. + */ + @AutoValue + public abstract static class Match { + + public abstract ValueString condition(); + + public abstract Result result(); + + /** Encapsulates the result of this match when condition is met. (either an output or a rule) */ + @AutoOneOf(Match.Result.Kind.class) + public abstract static class Result { + public abstract ValueString output(); + + public abstract Rule rule(); + + public abstract Kind kind(); + + public static Result ofOutput(ValueString value) { + return AutoOneOf_CelPolicy_Match_Result.output(value); + } + + public static Result ofRule(Rule value) { + return AutoOneOf_CelPolicy_Match_Result.rule(value); + } + + /** Kind for {@link Result}. */ + public enum Kind { + OUTPUT, + RULE + } + } + + /** Builder for {@link Match}. */ + @AutoValue.Builder + public abstract static class Builder implements RequiredFieldsChecker { + + public abstract Builder setCondition(ValueString condition); + + public abstract Builder setResult(Result result); + + abstract Optional result(); + + @Override + public ImmutableList requiredFields() { + return ImmutableList.of(RequiredField.of("output or a rule", this::result)); + } + + public abstract Match build(); + } + + /** Creates a new builder to construct a {@link Match} instance. */ + public static Builder newBuilder() { + return new AutoValue_CelPolicy_Match.Builder(); + } + } + + /** Variable is a named expression which may be referenced in subsequent expressions. */ + @AutoValue + public abstract static class Variable { + + public abstract ValueString name(); + + public abstract ValueString expression(); + + /** Builder for {@link Variable}. */ + @AutoValue.Builder + public abstract static class Builder { + + public abstract Builder setName(ValueString name); + + public abstract Builder setExpression(ValueString expression); + + public abstract Variable build(); + } + + /** Creates a new builder to construct a {@link Variable} instance. */ + public static Builder newBuilder() { + return new AutoValue_CelPolicy_Variable.Builder() + .setName(ValueString.newBuilder().build()) + .setExpression(ValueString.newBuilder().build()); + } + } +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java b/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java index 13f2bad66..c5acc43f5 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyConfig.java @@ -267,7 +267,7 @@ public CelFunctionDecl toCelFunctionDecl(CelTypeProvider celTypeProvider) { } } - /** Represents an overload declaraion on a policy function. */ + /** Represents an overload declaration on a policy function. */ @AutoValue public abstract static class OverloadDecl { diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyParser.java new file mode 100644 index 000000000..315ece06c --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyParser.java @@ -0,0 +1,87 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +/** CelPolicyParser is the interface for parsing policies into a canonical Policy representation. */ +public interface CelPolicyParser { + + /** Parses the input {@code policySource} and returns a {@link CelPolicy}. */ + CelPolicy parse(String policySource) throws CelPolicyValidationException; + + /** + * Parses the input {@code policySource} and returns a {@link CelPolicy}. + * + *

The {@code description} may be used to help tailor error messages for the location where the + * {@code policySource} originates, e.g. a file name or form UI element. + */ + CelPolicy parse(String policySource, String description) throws CelPolicyValidationException; + + /** + * TagVisitor declares a set of interfaces for handling custom tags which would otherwise be + * unsupported within the policy, rule, match, or variable objects. + * + * @param Type of the node (ex: YAML). + */ + interface TagVisitor { + + /** + * visitPolicyTag accepts a parser context, field id, tag name, yaml node, and parent Policy to + * allow for continued parsing within a custom tag. + */ + default void visitPolicyTag( + ParserContext ctx, long id, String fieldName, T node, CelPolicy.Builder policyBuilder) { + ctx.reportError(id, String.format("Unsupported policy tag: %s", fieldName)); + } + + /** + * visitRuleTag accepts a parser context, field id, tag name, yaml node, as well as the parent + * policy and current rule to allow for continued parsing within custom tags. + */ + default void visitRuleTag( + ParserContext ctx, + long id, + String fieldName, + T node, + CelPolicy.Rule.Builder ruleBuilder) { + ctx.reportError(id, String.format("Unsupported rule tag: %s", fieldName)); + } + + /** + * visitMatchTag accepts a parser context, field id, tag name, yaml node, as well as the parent + * policy and current match to allow for continued parsing within custom tags. + */ + default void visitMatchTag( + ParserContext ctx, + long id, + String fieldName, + T node, + CelPolicy.Match.Builder matchBuilder) { + ctx.reportError(id, String.format("Unsupported match tag: %s", fieldName)); + } + + /** + * visitVariableTag accepts a parser context, field id, tag name, yaml node, as well as the + * parent policy and current variable to allow for continued parsing within custom tags. + */ + default void visitVariableTag( + ParserContext ctx, + long id, + String fieldName, + T node, + CelPolicy.Variable.Builder variableBuilder) { + ctx.reportError(id, String.format("Unsupported variable tag: %s", fieldName)); + } + } +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyParserBuilder.java b/policy/src/main/java/dev/cel/policy/CelPolicyParserBuilder.java new file mode 100644 index 000000000..ba34a1a60 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyParserBuilder.java @@ -0,0 +1,35 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.errorprone.annotations.CheckReturnValue; +import dev.cel.policy.CelPolicyParser.TagVisitor; + +/** + * Interface for building an instance of {@link CelPolicyParser}. + * + * @param Type of the node (Ex: YAML). + */ +public interface CelPolicyParserBuilder { + + /** Adds a custom tag visitor to allow for handling of custom tags. */ + @CanIgnoreReturnValue + CelPolicyParserBuilder addTagVisitor(TagVisitor tagVisitor); + + /** Builds a new instance of {@link CelPolicyParser}. */ + @CheckReturnValue + CelPolicyParser build(); +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicySource.java b/policy/src/main/java/dev/cel/policy/CelPolicySource.java index 75565b563..c690652d9 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicySource.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicySource.java @@ -15,10 +15,12 @@ package dev.cel.policy; import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CheckReturnValue; import dev.cel.common.CelSourceHelper; import dev.cel.common.Source; import dev.cel.common.internal.CelCodePointArray; +import java.util.Map; import java.util.Optional; /** CelPolicySource represents the source content of a policy and its related metadata. */ @@ -31,6 +33,9 @@ public abstract class CelPolicySource implements Source { @Override public abstract String getDescription(); + @Override + public abstract ImmutableMap getPositionsMap(); + @Override public Optional getSnippet(int line) { return CelSourceHelper.getSnippet(getContent(), line); @@ -44,6 +49,8 @@ public abstract static class Builder { public abstract Builder setDescription(String description); + public abstract Builder setPositionsMap(Map value); + @CheckReturnValue public abstract CelPolicySource build(); } @@ -53,6 +60,7 @@ public abstract static class Builder { public static Builder newBuilder(CelCodePointArray celCodePointArray) { return new AutoValue_CelPolicySource.Builder() .setDescription("") - .setContent(celCodePointArray); + .setContent(celCodePointArray) + .setPositionsMap(ImmutableMap.of()); } } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java index 0cc348cc5..01dfc77b6 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java @@ -334,10 +334,15 @@ private CelPolicyConfig parseYaml(String source, String description) ParserContext ctx = YamlParserContextImpl.newInstance(configCodePointArray); CelPolicyConfig.Builder policyConfig = parseConfig(ctx, node); CelPolicySource configSource = - CelPolicySource.newBuilder(configCodePointArray).setDescription(description).build(); + CelPolicySource.newBuilder(configCodePointArray) + .setDescription(description) + .setPositionsMap(ctx.getIdToOffsetMap()) + .build(); + if (ctx.hasError()) { throw new CelPolicyValidationException(ctx.getIssueString(configSource)); } + return policyConfig.setConfigSource(configSource).build(); } @@ -347,6 +352,7 @@ private CelPolicyConfig.Builder parseConfig(ParserContext ctx, Node node) if (!assertYamlType(ctx, id, node, YamlNodeType.MAP)) { return builder; } + MappingNode rootNode = (MappingNode) node; for (NodeTuple nodeTuple : rootNode.getValue()) { Node keyNode = nodeTuple.getKeyNode(); @@ -354,6 +360,7 @@ private CelPolicyConfig.Builder parseConfig(ParserContext ctx, Node node) if (!assertYamlType(ctx, keyId, keyNode, YamlNodeType.STRING, YamlNodeType.TEXT)) { continue; } + Node valueNode = nodeTuple.getValueNode(); String fieldName = ((ScalarNode) keyNode).getValue(); switch (fieldName) { @@ -380,6 +387,7 @@ private CelPolicyConfig.Builder parseConfig(ParserContext ctx, Node node) // continue handling the rest of the nodes } } + return builder; } } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java new file mode 100644 index 000000000..64921bee9 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java @@ -0,0 +1,288 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.base.Preconditions.checkNotNull; +import static dev.cel.policy.YamlHelper.assertRequiredFields; +import static dev.cel.policy.YamlHelper.assertYamlType; +import static dev.cel.policy.YamlHelper.newValueString; + +import com.google.common.collect.ImmutableSet; +import dev.cel.common.internal.CelCodePointArray; +import dev.cel.policy.CelPolicy.Match; +import dev.cel.policy.CelPolicy.Variable; +import dev.cel.policy.YamlHelper.YamlNodeType; +import java.util.Optional; +import org.yaml.snakeyaml.nodes.MappingNode; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.NodeTuple; +import org.yaml.snakeyaml.nodes.ScalarNode; +import org.yaml.snakeyaml.nodes.SequenceNode; + +final class CelPolicyYamlParser implements CelPolicyParser { + + private final TagVisitor tagVisitor; + + @Override + public CelPolicy parse(String policySource) throws CelPolicyValidationException { + return parse(policySource, ""); + } + + @Override + public CelPolicy parse(String policySource, String description) + throws CelPolicyValidationException { + ParserImpl parser = new ParserImpl(tagVisitor); + return parser.parseYaml(policySource, description); + } + + private static class ParserImpl { + + private final TagVisitor tagVisitor; + + private CelPolicy parseYaml(String policySource, String description) + throws CelPolicyValidationException { + Node node; + try { + node = YamlHelper.parseYamlSource(policySource); + } catch (RuntimeException e) { + throw new CelPolicyValidationException("YAML document is malformed: " + e.getMessage(), e); + } + + CelCodePointArray policyCodePoints = CelCodePointArray.fromString(policySource); + ParserContext ctx = YamlParserContextImpl.newInstance(policyCodePoints); + + CelPolicy.Builder policyBuilder = parsePolicy(ctx, node); + CelPolicySource celPolicySource = + CelPolicySource.newBuilder(policyCodePoints) + .setDescription(description) + .setPositionsMap(ctx.getIdToOffsetMap()) + .build(); + + if (ctx.hasError()) { + throw new CelPolicyValidationException(ctx.getIssueString(celPolicySource)); + } + + return policyBuilder.setPolicySource(celPolicySource).build(); + } + + private CelPolicy.Builder parsePolicy(ParserContext ctx, Node node) { + CelPolicy.Builder policyBuilder = CelPolicy.newBuilder(); + long id = ctx.collectMetadata(node); + if (!assertYamlType(ctx, id, node, YamlNodeType.MAP)) { + return policyBuilder; + } + + MappingNode rootNode = (MappingNode) node; + for (NodeTuple nodeTuple : rootNode.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + if (!assertYamlType(ctx, keyId, keyNode, YamlNodeType.STRING, YamlNodeType.TEXT)) { + continue; + } + + Node valueNode = nodeTuple.getValueNode(); + String fieldName = ((ScalarNode) keyNode).getValue(); + switch (fieldName) { + case "name": + policyBuilder.setName(newValueString(ctx, valueNode)); + break; + case "rule": + policyBuilder.setRule(parseRule(ctx, valueNode)); + break; + default: + tagVisitor.visitPolicyTag(ctx, keyId, fieldName, valueNode, policyBuilder); + break; + } + } + + return policyBuilder; + } + + private CelPolicy.Rule parseRule(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + CelPolicy.Rule.Builder ruleBuilder = CelPolicy.Rule.newBuilder(); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.MAP)) { + return ruleBuilder.build(); + } + + for (NodeTuple nodeTuple : ((MappingNode) node).getValue()) { + Node key = nodeTuple.getKeyNode(); + long tagId = ctx.collectMetadata(key); + if (!assertYamlType(ctx, tagId, key, YamlNodeType.STRING, YamlNodeType.TEXT)) { + continue; + } + String fieldName = ((ScalarNode) key).getValue(); + Node value = nodeTuple.getValueNode(); + switch (fieldName) { + case "id": + ruleBuilder.setId(newValueString(ctx, value)); + break; + case "description": + ruleBuilder.setDescription(newValueString(ctx, value)); + break; + case "variables": + ruleBuilder.addVariables(parseVariables(ctx, value)); + break; + case "match": + ruleBuilder.addMatches(parseMatches(ctx, value)); + break; + default: + tagVisitor.visitRuleTag(ctx, tagId, fieldName, value, ruleBuilder); + break; + } + } + return ruleBuilder.build(); + } + + private ImmutableSet parseMatches(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + ImmutableSet.Builder matchesBuilder = ImmutableSet.builder(); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.LIST)) { + return matchesBuilder.build(); + } + + SequenceNode matchListNode = (SequenceNode) node; + for (Node elementNode : matchListNode.getValue()) { + parseMatch(ctx, elementNode).ifPresent(matchesBuilder::add); + } + + return matchesBuilder.build(); + } + + private Optional parseMatch(ParserContext ctx, Node node) { + long nodeId = ctx.collectMetadata(node); + if (!assertYamlType(ctx, nodeId, node, YamlNodeType.MAP)) { + return Optional.empty(); + } + MappingNode matchNode = (MappingNode) node; + CelPolicy.Match.Builder matchBuilder = + CelPolicy.Match.newBuilder().setCondition(ValueString.of(ctx.nextId(), "true")); + for (NodeTuple nodeTuple : matchNode.getValue()) { + Node key = nodeTuple.getKeyNode(); + long tagId = ctx.collectMetadata(key); + if (!assertYamlType(ctx, tagId, key, YamlNodeType.STRING, YamlNodeType.TEXT)) { + continue; + } + String fieldName = ((ScalarNode) key).getValue(); + Node value = nodeTuple.getValueNode(); + switch (fieldName) { + case "condition": + matchBuilder.setCondition(newValueString(ctx, value)); + break; + case "output": + matchBuilder + .result() + .filter(result -> result.kind().equals(Match.Result.Kind.RULE)) + .ifPresent( + result -> ctx.reportError(tagId, "Only the rule or the output may be set")); + matchBuilder.setResult(Match.Result.ofOutput(newValueString(ctx, value))); + break; + case "rule": + matchBuilder + .result() + .filter(result -> result.kind().equals(Match.Result.Kind.OUTPUT)) + .ifPresent( + result -> ctx.reportError(tagId, "Only the rule or the output may be set")); + matchBuilder.setResult(Match.Result.ofRule(parseRule(ctx, value))); + break; + default: + tagVisitor.visitMatchTag(ctx, tagId, fieldName, value, matchBuilder); + break; + } + } + + if (!assertRequiredFields(ctx, nodeId, matchBuilder.getMissingRequiredFieldNames())) { + return Optional.empty(); + } + + return Optional.of(matchBuilder.build()); + } + + private ImmutableSet parseVariables(ParserContext ctx, Node node) { + long valueId = ctx.collectMetadata(node); + ImmutableSet.Builder variableBuilder = ImmutableSet.builder(); + if (!assertYamlType(ctx, valueId, node, YamlNodeType.LIST)) { + return variableBuilder.build(); + } + + SequenceNode variableListNode = (SequenceNode) node; + for (Node elementNode : variableListNode.getValue()) { + long id = ctx.collectMetadata(elementNode); + if (!assertYamlType(ctx, id, elementNode, YamlNodeType.MAP)) { + continue; + } + variableBuilder.add(parseVariable(ctx, (MappingNode) elementNode)); + } + + return variableBuilder.build(); + } + + private CelPolicy.Variable parseVariable(ParserContext ctx, MappingNode variableMap) { + Variable.Builder builder = Variable.newBuilder(); + + for (NodeTuple nodeTuple : variableMap.getValue()) { + Node keyNode = nodeTuple.getKeyNode(); + long keyId = ctx.collectMetadata(keyNode); + Node valueNode = nodeTuple.getValueNode(); + String keyName = ((ScalarNode) keyNode).getValue(); + switch (keyName) { + case "name": + builder.setName(newValueString(ctx, valueNode)); + break; + case "expression": + builder.setExpression(newValueString(ctx, valueNode)); + break; + default: + tagVisitor.visitVariableTag(ctx, keyId, keyName, valueNode, builder); + break; + } + } + + return builder.build(); + } + + private ParserImpl(TagVisitor tagVisitor) { + this.tagVisitor = tagVisitor; + } + } + + static final class Builder implements CelPolicyParserBuilder { + + private TagVisitor tagVisitor; + + private Builder() { + this.tagVisitor = new TagVisitor() {}; + } + + @Override + public CelPolicyParserBuilder addTagVisitor(TagVisitor tagVisitor) { + this.tagVisitor = tagVisitor; + return this; + } + + @Override + public CelPolicyParser build() { + return new CelPolicyYamlParser(tagVisitor); + } + } + + static Builder newBuilder() { + return new Builder(); + } + + private CelPolicyYamlParser(TagVisitor tagVisitor) { + this.tagVisitor = checkNotNull(tagVisitor); + } +} diff --git a/policy/src/main/java/dev/cel/policy/ParserContext.java b/policy/src/main/java/dev/cel/policy/ParserContext.java index ec4c72f20..3b68f82a5 100644 --- a/policy/src/main/java/dev/cel/policy/ParserContext.java +++ b/policy/src/main/java/dev/cel/policy/ParserContext.java @@ -15,6 +15,7 @@ package dev.cel.policy; import dev.cel.common.Source; +import java.util.Map; /** * ParserContext declares a set of interfaces for creating and managing metadata for parsed @@ -30,4 +31,6 @@ public interface ParserContext { String getIssueString(Source source); boolean hasError(); + + Map getIdToOffsetMap(); } diff --git a/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java index 34f70db6c..31438fd98 100644 --- a/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java +++ b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java @@ -23,6 +23,7 @@ import dev.cel.common.internal.CelCodePointArray; import java.util.ArrayList; import java.util.HashMap; +import java.util.Map; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.ScalarNode; @@ -54,6 +55,11 @@ public boolean hasError() { return !issues.isEmpty(); } + @Override + public Map getIdToOffsetMap() { + return idToOffsetMap; + } + @Override public long collectMetadata(Node node) { long id = nextId(); diff --git a/policy/src/test/java/dev/cel/policy/BUILD.bazel b/policy/src/test/java/dev/cel/policy/BUILD.bazel index b4844833a..38e88e5f4 100644 --- a/policy/src/test/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/test/java/dev/cel/policy/BUILD.bazel @@ -6,16 +6,22 @@ java_library( name = "tests", testonly = True, srcs = glob(["*.java"]), + resources = [ + "//policy/src/test/resources:policy_yaml_files", + ], deps = [ "//:java_truth", "//bundle:cel", "//common:options", "//common/internal", + "//policy", "//policy:config", "//policy:config_parser", + "//policy:parser", "//policy:source", "//policy:validation_exception", "//policy:yaml_config_parser", + "//policy:yaml_parser", "@maven//:com_google_api_grpc_proto_google_common_protos", "@maven//:com_google_guava_guava", "@maven//:com_google_testparameterinjector_test_parameter_injector", diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java new file mode 100644 index 000000000..fc19d8f0c --- /dev/null +++ b/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java @@ -0,0 +1,202 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.truth.Truth.assertThat; +import static org.junit.Assert.assertThrows; + +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.policy.PolicyTestHelper.TestYamlPolicy; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public final class CelPolicyYamlParserTest { + + private static final CelPolicyParser POLICY_PARSER = CelPolicyYamlParser.newBuilder().build(); + + @Test + public void parseYamlPolicy_success(@TestParameter TestYamlPolicy yamlPolicy) throws Exception { + String policySource = yamlPolicy.readPolicyYamlContent(); + String description = yamlPolicy.getPolicyName(); + + CelPolicy policy = POLICY_PARSER.parse(policySource, description); + + assertThat(policy.name().value()).isEqualTo(yamlPolicy.getPolicyName()); + assertThat(policy.policySource().getContent().toString()).isEqualTo(policySource); + assertThat(policy.policySource().getDescription()).isEqualTo(description); + } + + @Test + public void parseYamlPolicy_errors(@TestParameter PolicyParseErrorTestCase testCase) { + CelPolicyValidationException e = + assertThrows( + CelPolicyValidationException.class, () -> POLICY_PARSER.parse(testCase.yamlPolicy)); + assertThat(e).hasMessageThat().isEqualTo(testCase.expectedErrorMessage); + } + + private enum PolicyParseErrorTestCase { + MALFORMED_YAML_DOCUMENT( + "a:\na", + "YAML document is malformed: while scanning a simple key\n" + + " in 'reader', line 2, column 1:\n" + + " a\n" + + " ^\n" + + "could not find expected ':'\n" + + " in 'reader', line 2, column 2:\n" + + " a\n" + + " ^\n"), + ILLEGAL_YAML_TYPE_POLICY_KEY( + "1: test", + "ERROR: :1:1: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:str !txt]\n" + + " | 1: test\n" + + " | ^"), + ILLEGAL_YAML_TYPE_ON_NAME_VALUE( + "name: \n" + " illegal: yaml-type", + "ERROR: :2:3: Got yaml node type tag:yaml.org,2002:map, wanted type(s)" + + " [tag:yaml.org,2002:str !txt]\n" + + " | illegal: yaml-type\n" + + " | ..^"), + ILLEGAL_YAML_TYPE_ON_RULE_VALUE( + "rule: illegal", + "ERROR: :1:7: Got yaml node type tag:yaml.org,2002:str, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | rule: illegal\n" + + " | ......^"), + ILLEGAL_YAML_TYPE_ON_RULE_MAP_KEY( + "rule: \n" + " 1: foo", + "ERROR: :2:3: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:str !txt]\n" + + " | 1: foo\n" + + " | ..^"), + ILLEGAL_YAML_TYPE_ON_MATCHES_VALUE( + "rule:\n" + " match: illegal\n", + "ERROR: :2:10: Got yaml node type tag:yaml.org,2002:str, wanted type(s)" + + " [tag:yaml.org,2002:seq]\n" + + " | match: illegal\n" + + " | .........^"), + ILLEGAL_YAML_TYPE_ON_MATCHES_LIST( + "rule:\n" + " match:\n" + " - illegal", + "ERROR: :3:7: Got yaml node type tag:yaml.org,2002:str, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - illegal\n" + + " | ......^"), + ILLEGAL_YAML_TYPE_ON_MATCH_MAP_KEY( + "rule:\n" + " match:\n" + " - 1 : foo\n" + " output: 'hi'", + "ERROR: :3:7: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:str !txt]\n" + + " | - 1 : foo\n" + + " | ......^"), + ILLEGAL_YAML_TYPE_ON_VARIABLE_VALUE( + "rule:\n" + " variables: illegal\n", + "ERROR: :2:14: Got yaml node type tag:yaml.org,2002:str, wanted type(s)" + + " [tag:yaml.org,2002:seq]\n" + + " | variables: illegal\n" + + " | .............^"), + ILLEGAL_YAML_TYPE_ON_VARIABLE_MAP_KEY( + "rule:\n" + " variables:\n" + " - illegal", + "ERROR: :3:7: Got yaml node type tag:yaml.org,2002:str, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - illegal\n" + + " | ......^"), + MULTIPLE_YAML_DOCS( + "name: foo\n" + "---\n" + "name: bar", + "YAML document is malformed: expected a single document in the stream\n" + + " in 'reader', line 1, column 1:\n" + + " name: foo\n" + + " ^\n" + + "but found another document\n" + + " in 'reader', line 2, column 1:\n" + + " ---\n" + + " ^\n"), + UNSUPPORTED_RULE_TAG( + "rule:\n" + " custom: yaml-type", + "ERROR: :2:3: Unsupported rule tag: custom\n" + + " | custom: yaml-type\n" + + " | ..^"), + UNSUPPORTED_POLICY_TAG( + "inputs:\n" + " - name: a\n" + " - name: b", + "ERROR: :1:1: Unsupported policy tag: inputs\n" + " | inputs:\n" + " | ^"), + UNSUPPORTED_VARIABLE_TAG( + "rule:\n" + " variables:\n" + " - name: 'true'\n" + " alt_name: 'bool_true'", + "ERROR: :4:7: Unsupported variable tag: alt_name\n" + + " | alt_name: 'bool_true'\n" + + " | ......^"), + UNSUPPORTED_MATCH_TAG( + "rule:\n" + + " match:\n" + + " - name: 'true'\n" + + " output: 'hi'\n" + + " alt_name: 'bool_true'", + "ERROR: :3:7: Unsupported match tag: name\n" + + " | - name: 'true'\n" + + " | ......^\n" + + "ERROR: :5:7: Unsupported match tag: alt_name\n" + + " | alt_name: 'bool_true'\n" + + " | ......^"), + MATCH_MISSING_OUTPUT_AND_RULE( + "rule:\n" // + + " match:\n" // + + " - condition: 'true'", + "ERROR: :3:7: Missing required attribute(s): output or a rule\n" + + " | - condition: 'true'\n" + + " | ......^"), + MATCH_OUTPUT_SET_THEN_RULE( + "rule:\n" + + " match:\n" + + " - condition: \"true\"\n" + + " output: \"world\"\n" + + " rule:\n" + + " match:\n" + + " - output: \"hello\"", + "ERROR: :5:7: Only the rule or the output may be set\n" + + " | rule:\n" + + " | ......^"), + MATCH_RULE_SET_THEN_OUTPUT( + "rule:\n" + + " match:\n" + + " - condition: \"true\"\n" + + " rule:\n" + + " match:\n" + + " - output: \"hello\"\n" + + " output: \"world\"", + "ERROR: :7:7: Only the rule or the output may be set\n" + + " | output: \"world\"\n" + + " | ......^"), + INVALID_ROOT_NODE_TYPE( + "- rule:\n" + " id: a", + "ERROR: :1:1: Got yaml node type tag:yaml.org,2002:seq, wanted type(s)" + + " [tag:yaml.org,2002:map]\n" + + " | - rule:\n" + + " | ^"), + ILLEGAL_RULE_DESCRIPTION_TYPE( + "rule:\n" + " description: 1", + "ERROR: :2:16: Got yaml node type tag:yaml.org,2002:int, wanted type(s)" + + " [tag:yaml.org,2002:str !txt]\n" + + " | description: 1\n" + + " | ...............^"), + ; + + private final String yamlPolicy; + private final String expectedErrorMessage; + + PolicyParseErrorTestCase(String yamlPolicy, String expectedErrorMessage) { + this.yamlPolicy = yamlPolicy; + this.expectedErrorMessage = expectedErrorMessage; + } + } +} diff --git a/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java b/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java new file mode 100644 index 000000000..c04d15440 --- /dev/null +++ b/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java @@ -0,0 +1,105 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import com.google.common.base.Ascii; +import com.google.common.io.Resources; +import java.io.IOException; +import java.net.URL; + +/** Package-private class to assist with policy testing. */ +final class PolicyTestHelper { + + enum TestYamlPolicy { + NESTED_RULE( + "nested_rule", + true, + "cel.bind(variables.permitted_regions, [\"us\", \"uk\", \"es\"]," + + " cel.bind(variables.banned_regions, {\"us\": false, \"ru\": false, \"ir\": false}," + + " (resource.origin in variables.banned_regions && " + + "!(resource.origin in variables.permitted_regions)) " + + "? optional.of({\"banned\": true}) : optional.none()).or(" + + "optional.of((resource.origin in variables.permitted_regions)" + + " ? {\"banned\": false} : {\"banned\": true})))"), + REQUIRED_LABELS( + "required_labels", + true, + "" + + "cel.bind(variables.want, spec.labels, cel.bind(variables.missing, " + + "variables.want.filter(l, !(l in resource.labels)), cel.bind(variables.invalid, " + + "resource.labels.filter(l, l in variables.want && variables.want[l] != " + + "resource.labels[l]), (variables.missing.size() > 0) ? " + + "optional.of(\"missing one or more required labels: [\"\" + " + + "variables.missing.join(\",\") + \"\"]\") : ((variables.invalid.size() > 0) ? " + + "optional.of(\"invalid values provided on one or more labels: [\"\" + " + + "variables.invalid.join(\",\") + \"\"]\") : optional.none()))))"), + RESTRICTED_DESTINATIONS( + "restricted_destinations", + false, + "cel.bind(variables.matches_origin_ip, locationCode(origin.ip) == spec.origin," + + " cel.bind(variables.has_nationality, has(request.auth.claims.nationality)," + + " cel.bind(variables.matches_nationality, variables.has_nationality &&" + + " request.auth.claims.nationality == spec.origin, cel.bind(variables.matches_dest_ip," + + " locationCode(destination.ip) in spec.restricted_destinations," + + " cel.bind(variables.matches_dest_label, resource.labels.location in" + + " spec.restricted_destinations, cel.bind(variables.matches_dest," + + " variables.matches_dest_ip || variables.matches_dest_label," + + " (variables.matches_nationality && variables.matches_dest) ? true :" + + " ((!variables.has_nationality && variables.matches_origin_ip &&" + + " variables.matches_dest) ? true : false)))))))"); + + private final String name; + private final boolean producesOptionalResult; + private final String unparsed; + + TestYamlPolicy(String name, boolean producesOptionalResult, String unparsed) { + this.name = name; + this.producesOptionalResult = producesOptionalResult; + this.unparsed = unparsed; + } + + String getPolicyName() { + return name; + } + + boolean producesOptionalResult() { + return this.producesOptionalResult; + } + + String getUnparsed() { + return unparsed; + } + + String readPolicyYamlContent() throws IOException { + return readFromYaml(String.format("%s/policy.yaml", name)); + } + } + + static String readFromYaml(String yamlPath) throws IOException { + return readFile(yamlPath); + } + + private static URL getResource(String path) { + return Resources.getResource(Ascii.toLowerCase(path)); + } + + private static String readFile(String path) throws IOException { + return Resources.toString(getResource(path), UTF_8); + } + + private PolicyTestHelper() {} +} From 3d2aadabf890fbf2a09d4f697850b0dc7e24e1c7 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Tue, 2 Jul 2024 15:55:09 -0700 Subject: [PATCH 17/33] Move computing offset location into CelSourceHelper for reuse PiperOrigin-RevId: 648869686 --- .../src/main/java/dev/cel/common/BUILD.bazel | 1 + .../main/java/dev/cel/common/CelSource.java | 39 +------------------ .../java/dev/cel/common/CelSourceHelper.java | 34 ++++++++++++++++ .../src/main/java/dev/cel/policy/BUILD.bazel | 1 + .../java/dev/cel/policy/CelPolicySource.java | 8 ++++ 5 files changed, 46 insertions(+), 37 deletions(-) diff --git a/common/src/main/java/dev/cel/common/BUILD.bazel b/common/src/main/java/dev/cel/common/BUILD.bazel index 82e74bf1d..dbb8890d5 100644 --- a/common/src/main/java/dev/cel/common/BUILD.bazel +++ b/common/src/main/java/dev/cel/common/BUILD.bazel @@ -203,6 +203,7 @@ java_library( name = "source", srcs = SOURCE_SOURCES, deps = [ + ":source_location", "//common/annotations", "//common/internal", "@maven//:com_google_guava_guava", diff --git a/common/src/main/java/dev/cel/common/CelSource.java b/common/src/main/java/dev/cel/common/CelSource.java index 574946fd3..cc9244e30 100644 --- a/common/src/main/java/dev/cel/common/CelSource.java +++ b/common/src/main/java/dev/cel/common/CelSource.java @@ -108,7 +108,7 @@ public Optional getLocationOffset(int line, int column) { * Get the line and column in the source expression text for the given code point {@code offset}. */ public Optional getOffsetLocation(int offset) { - return getOffsetLocationImpl(lineOffsets, offset); + return CelSourceHelper.getOffsetLocation(codePoints, offset); } @Override @@ -134,30 +134,6 @@ private static Optional getLocationOffsetImpl( return Optional.of(offset + column); } - /** - * Get the line and column in the source expression text for the given code point {@code offset}. - */ - public static Optional getOffsetLocationImpl( - List lineOffsets, int offset) { - checkArgument(offset >= 0); - LineAndOffset lineAndOffset = findLine(lineOffsets, offset); - return Optional.of(CelSourceLocation.of(lineAndOffset.line, offset - lineAndOffset.offset)); - } - - private static LineAndOffset findLine(List lineOffsets, int offset) { - int line = 1; - for (int index = 0; index < lineOffsets.size(); index++) { - if (lineOffsets.get(index) > offset) { - break; - } - line++; - } - if (line == 1) { - return new LineAndOffset(line, 0); - } - return new LineAndOffset(line, lineOffsets.get(line - 2)); - } - public Builder toBuilder() { return new Builder(codePoints, lineOffsets) .setDescription(description) @@ -311,7 +287,7 @@ public Optional getLocationOffset(int line, int column) { * offset}. */ public Optional getOffsetLocation(int offset) { - return getOffsetLocationImpl(lineOffsets, offset); + return CelSourceHelper.getOffsetLocation(codePoints, offset); } @CheckReturnValue @@ -335,17 +311,6 @@ public CelSource build() { } } - private static final class LineAndOffset { - - private LineAndOffset(int line, int offset) { - this.line = line; - this.offset = offset; - } - - private final int line; - private final int offset; - } - /** * Tag for an extension that were used while parsing or type checking the source expression. For * example, optimizations that require special runtime support may be specified. These are used to diff --git a/common/src/main/java/dev/cel/common/CelSourceHelper.java b/common/src/main/java/dev/cel/common/CelSourceHelper.java index 4fde5db7b..13168edbe 100644 --- a/common/src/main/java/dev/cel/common/CelSourceHelper.java +++ b/common/src/main/java/dev/cel/common/CelSourceHelper.java @@ -47,6 +47,40 @@ public static Optional getSnippet(CelCodePointArray content, int line) { return Optional.of(end != start ? content.slice(start, end).toString() : ""); } + /** + * Get the line and column in the source expression text for the given code point {@code offset}. + */ + public static Optional getOffsetLocation( + CelCodePointArray content, int offset) { + checkArgument(offset >= 0); + LineAndOffset lineAndOffset = findLine(content.lineOffsets(), offset); + return Optional.of(CelSourceLocation.of(lineAndOffset.line, offset - lineAndOffset.offset)); + } + + private static LineAndOffset findLine(List lineOffsets, int offset) { + int line = 1; + for (Integer lineOffset : lineOffsets) { + if (lineOffset > offset) { + break; + } + line++; + } + if (line == 1) { + return new LineAndOffset(line, 0); + } + return new LineAndOffset(line, lineOffsets.get(line - 2)); + } + + private static final class LineAndOffset { + private LineAndOffset(int line, int offset) { + this.line = line; + this.offset = offset; + } + + private final int line; + private final int offset; + } + static int findLineOffset(List lineOffsets, int line) { if (line == 1) { return 0; diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index ed152b9f2..0c718d727 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -30,6 +30,7 @@ java_library( deps = [ "//:auto_value", "//common:source", + "//common:source_location", "//common/internal", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", diff --git a/policy/src/main/java/dev/cel/policy/CelPolicySource.java b/policy/src/main/java/dev/cel/policy/CelPolicySource.java index c690652d9..7918be872 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicySource.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicySource.java @@ -18,6 +18,7 @@ import com.google.common.collect.ImmutableMap; import com.google.errorprone.annotations.CheckReturnValue; import dev.cel.common.CelSourceHelper; +import dev.cel.common.CelSourceLocation; import dev.cel.common.Source; import dev.cel.common.internal.CelCodePointArray; import java.util.Map; @@ -41,6 +42,13 @@ public Optional getSnippet(int line) { return CelSourceHelper.getSnippet(getContent(), line); } + /** + * Get the line and column in the source expression text for the given code point {@code offset}. + */ + public Optional getOffsetLocation(int offset) { + return CelSourceHelper.getOffsetLocation(getContent(), offset); + } + /** Builder for {@link CelPolicySource}. */ @AutoValue.Builder public abstract static class Builder { From 6399ae54370a08b68f11be861dd6cad2f265f092 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Tue, 2 Jul 2024 17:06:57 -0700 Subject: [PATCH 18/33] Add AstMutator methods to construct cel.bind macro with varInit containing macros PiperOrigin-RevId: 648887898 --- .../java/dev/cel/optimizer/AstMutator.java | 32 ++++++-- .../dev/cel/optimizer/AstMutatorTest.java | 75 +++++++++++++++++++ 2 files changed, 102 insertions(+), 5 deletions(-) diff --git a/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java b/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java index 8d35efc2c..dfcdb1a05 100644 --- a/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java +++ b/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java @@ -111,16 +111,17 @@ public CelMutableAst wrapAstWithNewCelBlock( public CelMutableAst replaceSubtreeWithNewBindMacro( CelMutableAst ast, String varName, - CelMutableExpr varInit, + CelMutableAst varInit, CelMutableExpr resultExpr, long exprIdToReplace, boolean populateMacroSource) { - // Copy the incoming expressions to prevent modifying the root - long maxId = max(getMaxId(varInit), getMaxId(ast)); + // Stabilize incoming varInit AST to avoid collision with the main AST + long maxId = getMaxId(ast); + varInit = stabilizeAst(varInit, maxId); StableIdGenerator stableIdGenerator = CelExprIdGeneratorFactory.newStableIdGenerator(maxId); CelMutableExpr newBindMacroExpr = newBindMacroExpr( - varName, varInit, CelMutableExpr.newInstance(resultExpr), stableIdGenerator); + varName, varInit.expr(), CelMutableExpr.newInstance(resultExpr), stableIdGenerator); CelMutableSource celSource = CelMutableSource.newInstance(); if (populateMacroSource) { CelMutableExpr newBindMacroSourceExpr = @@ -128,9 +129,10 @@ public CelMutableAst replaceSubtreeWithNewBindMacro( // In situations where the existing AST already contains a macro call (ex: nested cel.binds), // its macro source must be normalized to make it consistent with the newly generated bind // macro. + celSource = combine(ast.source(), varInit.source()); celSource = normalizeMacroSource( - ast.source(), + celSource, -1, // Do not replace any of the subexpr in the macro map. newBindMacroSourceExpr, stableIdGenerator::renumberId); @@ -142,6 +144,26 @@ public CelMutableAst replaceSubtreeWithNewBindMacro( return replaceSubtree(ast, newBindAst, exprIdToReplace); } + /** + * See {@link #replaceSubtreeWithNewBindMacro(CelMutableAst, String, CelMutableAst, + * CelMutableExpr, long, boolean)}. + */ + public CelMutableAst replaceSubtreeWithNewBindMacro( + CelMutableAst ast, + String varName, + CelMutableExpr varInit, + CelMutableExpr resultExpr, + long exprIdToReplace, + boolean populateMacroSource) { + return replaceSubtreeWithNewBindMacro( + ast, + varName, + CelMutableAst.of(varInit, CelMutableSource.newInstance()), + resultExpr, + exprIdToReplace, + populateMacroSource); + } + /** Renumbers all the expr IDs in the given AST in a consecutive manner starting from 1. */ public CelMutableAst renumberIdsConsecutively(CelMutableAst mutableAst) { StableIdGenerator stableIdGenerator = CelExprIdGeneratorFactory.newStableIdGenerator(0); diff --git a/optimizer/src/test/java/dev/cel/optimizer/AstMutatorTest.java b/optimizer/src/test/java/dev/cel/optimizer/AstMutatorTest.java index 8d56c2a3a..d9d9a969e 100644 --- a/optimizer/src/test/java/dev/cel/optimizer/AstMutatorTest.java +++ b/optimizer/src/test/java/dev/cel/optimizer/AstMutatorTest.java @@ -446,6 +446,81 @@ public void replaceSubtreeWithNewBindMacro_replaceRootWithNestedBindMacro() thro assertConsistentMacroCalls(mutatedAst); } + @Test + public void replaceSubtreeWithNewBindMacro_varInitContainsMacro_replaceRoot() throws Exception { + // Arrange + CelAbstractSyntaxTree ast = CEL.compile("true && true").getAst(); + CelMutableAst mutableAst = CelMutableAst.fromCelAst(ast); + CelMutableAst varInitAst = CelMutableAst.fromCelAst(CEL.compile("[].exists(x, x)").getAst()); + String variableName = "@r0"; + CelMutableExpr resultExpr = + CelMutableExpr.ofCall( + CelMutableCall.create( + Operator.LOGICAL_AND.getFunction(), + CelMutableExpr.ofIdent(variableName), + CelMutableExpr.ofIdent(variableName))); + + // Act + // Perform the initial replacement. (true && true) -> cel.bind(@r0, [].exists(x, x), @r0 && @r0) + mutableAst = + AST_MUTATOR.replaceSubtreeWithNewBindMacro( + mutableAst, + variableName, + varInitAst, + resultExpr, + ast.getExpr().id(), + true); // Replace && + + CelAbstractSyntaxTree mutatedAst = mutableAst.toParsedAst(); + assertThat(mutatedAst.getSource().getMacroCalls()).hasSize(2); + assertThat(CEL.createProgram(CEL.check(mutatedAst).getAst()).eval()).isEqualTo(false); + assertThat(CEL_UNPARSER.unparse(mutatedAst)) + .isEqualTo("cel.bind(@r0, [].exists(x, x), @r0 && @r0)"); + assertConsistentMacroCalls(mutatedAst); + } + + @Test + public void replaceSubtreeWithNewBindMacro_astAndVarInitContainsMacro_replaceRhs() + throws Exception { + // Arrange + CelAbstractSyntaxTree ast = CEL.compile("[true].exists(y,y) && false").getAst(); + CelMutableAst mutableAst = CelMutableAst.fromCelAst(ast); + CelMutableAst varInitAst = CelMutableAst.fromCelAst(CEL.compile("[].exists(x, x)").getAst()); + String variableName = "@r0"; + CelMutableExpr resultExpr = CelMutableExpr.ofIdent(variableName); + long falseExprId = + CelNavigableAst.fromAst(ast) + .getRoot() + .children() + .map(CelNavigableExpr::expr) + .filter( + x -> + x.getKind().equals(Kind.CONSTANT) + && x.constant().getKind().equals(CelConstant.Kind.BOOLEAN_VALUE)) + .filter(x -> !x.constant().booleanValue()) + .findAny() + .get() + .id(); + + // Act + // Perform the initial replacement. (true && true) -> cel.bind(@r0, [].exists(x, x), @r0 && @r0) + mutableAst = + AST_MUTATOR.replaceSubtreeWithNewBindMacro( + mutableAst, + variableName, + varInitAst, + resultExpr, + falseExprId, // Replace false + true); + + CelAbstractSyntaxTree mutatedAst = mutableAst.toParsedAst(); + assertThat(mutatedAst.getSource().getMacroCalls()).hasSize(3); + assertThat(CEL.createProgram(CEL.check(mutatedAst).getAst()).eval()).isEqualTo(false); + assertThat(CEL_UNPARSER.unparse(mutatedAst)) + .isEqualTo("[true].exists(y, y) && cel.bind(@r0, [].exists(x, x), @r0)"); + assertConsistentMacroCalls(mutatedAst); + } + @Test public void replaceSubtree_macroReplacedWithConstExpr_macroCallCleared() throws Exception { CelAbstractSyntaxTree ast = From cc70add6424ea1d06c6e76e3e9a123ea44ca530b Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Tue, 2 Jul 2024 17:17:45 -0700 Subject: [PATCH 19/33] Add AstMutator methods to construct function calls PiperOrigin-RevId: 648890320 --- .../java/dev/cel/optimizer/AstMutator.java | 66 +++++++++++++++++++ .../dev/cel/optimizer/AstMutatorTest.java | 23 +++++++ 2 files changed, 89 insertions(+) diff --git a/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java b/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java index dfcdb1a05..f9c9db9c6 100644 --- a/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java +++ b/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java @@ -40,6 +40,8 @@ import dev.cel.common.navigation.TraversalOrder; import dev.cel.common.types.CelType; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -95,6 +97,70 @@ public CelMutableAst wrapAstWithNewCelBlock( return CelMutableAst.of(blockExpr, ast.source()); } + /** + * Constructs a new global call wrapped in an AST with the provided ASTs as its argument. This + * will preserve all macro source information contained within the arguments. + */ + public CelMutableAst newGlobalCall(String function, Collection args) { + return newCallAst(Optional.empty(), function, args); + } + + /** + * Constructs a new global call wrapped in an AST with the provided ASTs as its argument. This + * will preserve all macro source information contained within the arguments. + */ + public CelMutableAst newGlobalCall(String function, CelMutableAst... args) { + return newGlobalCall(function, Arrays.asList(args)); + } + + /** + * Constructs a new member call wrapped in an AST the provided ASTs as its arguments. This will + * preserve all macro source information contained within the arguments. + */ + public CelMutableAst newMemberCall(CelMutableAst target, String function, CelMutableAst... args) { + return newMemberCall(target, function, Arrays.asList(args)); + } + + /** + * Constructs a new member call wrapped in an AST the provided ASTs as its arguments. This will + * preserve all macro source information contained within the arguments. + */ + public CelMutableAst newMemberCall( + CelMutableAst target, String function, Collection args) { + return newCallAst(Optional.of(target), function, args); + } + + private CelMutableAst newCallAst( + Optional target, String function, Collection args) { + long maxId = 1; + CelMutableSource combinedSource = CelMutableSource.newInstance(); + for (CelMutableAst arg : args) { + CelMutableAst stableArg = stabilizeAst(arg, maxId); + maxId = getMaxId(stableArg); + combinedSource = combine(combinedSource, stableArg.source()); + } + + Optional maybeTarget = Optional.empty(); + if (target.isPresent()) { + CelMutableAst stableTarget = stabilizeAst(target.get(), maxId); + combinedSource = combine(combinedSource, stableTarget.source()); + maxId = getMaxId(stableTarget); + + maybeTarget = Optional.of(stableTarget); + } + + List exprArgs = + args.stream().map(CelMutableAst::expr).collect(toCollection(ArrayList::new)); + CelMutableCall newCall = + maybeTarget + .map(celMutableAst -> CelMutableCall.create(celMutableAst.expr(), function, exprArgs)) + .orElseGet(() -> CelMutableCall.create(function, exprArgs)); + + CelMutableExpr newCallExpr = CelMutableExpr.ofCall(++maxId, newCall); + + return CelMutableAst.of(newCallExpr, combinedSource); + } + /** * Generates a new bind macro using the provided initialization and result expression, then * replaces the subtree using the new bind expr at the designated expr ID. diff --git a/optimizer/src/test/java/dev/cel/optimizer/AstMutatorTest.java b/optimizer/src/test/java/dev/cel/optimizer/AstMutatorTest.java index d9d9a969e..cdfa1cb00 100644 --- a/optimizer/src/test/java/dev/cel/optimizer/AstMutatorTest.java +++ b/optimizer/src/test/java/dev/cel/optimizer/AstMutatorTest.java @@ -1109,6 +1109,29 @@ public void replaceSubtree_iterationLimitReached_throws() throws Exception { assertThat(e).hasMessageThat().isEqualTo("Max iteration count reached."); } + @Test + public void newGlobalCallAst_success() throws Exception { + CelMutableAst argAst1 = CelMutableAst.fromCelAst(CEL.compile("[1].exists(x, x >= 1)").getAst()); + CelMutableAst argAst2 = CelMutableAst.fromCelAst(CEL.compile("'hello'").getAst()); + + CelMutableAst callAst = AST_MUTATOR.newGlobalCall("func", argAst1, argAst2); + + assertThat(CEL_UNPARSER.unparse(callAst.toParsedAst())) + .isEqualTo("func([1].exists(x, x >= 1), \"hello\")"); + } + + @Test + public void newMemberCallAst_success() throws Exception { + CelMutableAst targetAst = CelMutableAst.fromCelAst(CEL.compile("'hello'").getAst()); + CelMutableAst argAst1 = CelMutableAst.fromCelAst(CEL.compile("[1].exists(x, x >= 1)").getAst()); + CelMutableAst argAst2 = CelMutableAst.fromCelAst(CEL.compile("'world'").getAst()); + + CelMutableAst callAst = AST_MUTATOR.newMemberCall(targetAst, "func", argAst1, argAst2); + + assertThat(CEL_UNPARSER.unparse(callAst.toParsedAst())) + .isEqualTo("\"hello\".func([1].exists(x, x >= 1), \"world\")"); + } + /** * Asserts that the expressions that appears in source_info's macro calls are consistent with the * actual expr nodes in the AST. From 9fa219d072d2de7875555c46b8cf098c76484580 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Wed, 3 Jul 2024 13:13:11 -0700 Subject: [PATCH 20/33] Add Policy Compiler PiperOrigin-RevId: 649176985 --- .../java/dev/cel/optimizer/AstMutator.java | 2 +- policy/BUILD.bazel | 6 + .../src/main/java/dev/cel/policy/BUILD.bazel | 96 ++++++- .../java/dev/cel/policy/CelCompiledRule.java | 83 ++++++ .../dev/cel/policy/CelPolicyCompiler.java | 27 ++ .../cel/policy/CelPolicyCompilerBuilder.java | 28 +++ .../dev/cel/policy/CelPolicyCompilerImpl.java | 215 ++++++++++++++++ .../java/dev/cel/policy/RuleComposer.java | 135 ++++++++++ .../src/test/java/dev/cel/policy/BUILD.bazel | 7 + .../cel/policy/CelPolicyCompilerImplTest.java | 237 ++++++++++++++++++ .../java/dev/cel/policy/PolicyTestHelper.java | 120 +++++++++ policy/src/test/resources/errors/policy.yaml | 3 + 12 files changed, 951 insertions(+), 8 deletions(-) create mode 100644 policy/src/main/java/dev/cel/policy/CelCompiledRule.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyCompilerBuilder.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java create mode 100644 policy/src/main/java/dev/cel/policy/RuleComposer.java create mode 100644 policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java diff --git a/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java b/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java index f9c9db9c6..aa90f1528 100644 --- a/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java +++ b/optimizer/src/main/java/dev/cel/optimizer/AstMutator.java @@ -132,7 +132,7 @@ public CelMutableAst newMemberCall( private CelMutableAst newCallAst( Optional target, String function, Collection args) { - long maxId = 1; + long maxId = 0; CelMutableSource combinedSource = CelMutableSource.newInstance(); for (CelMutableAst arg : args) { CelMutableAst stableArg = stabilizeAst(arg, maxId); diff --git a/policy/BUILD.bazel b/policy/BUILD.bazel index 8e65f4229..2781b5663 100644 --- a/policy/BUILD.bazel +++ b/policy/BUILD.bazel @@ -33,6 +33,12 @@ java_library( exports = ["//policy/src/main/java/dev/cel/policy:parser"], ) +java_library( + name = "compiler_impl", + visibility = ["//visibility:public"], + exports = ["//policy/src/main/java/dev/cel/policy:compiler_impl"], +) + java_library( name = "yaml_parser", visibility = ["//visibility:public"], diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index 0c718d727..bf755f2e9 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -89,11 +89,11 @@ java_library( "CelPolicyYamlParser.java", ], deps = [ + ":common_internal", ":parser", + ":parser_builder", ":parser_context", ":policy", - ":policy_common_internal", - ":policy_parser_builder", ":source", ":validation_exception", ":value_string", @@ -116,7 +116,7 @@ java_library( ) java_library( - name = "policy_parser_builder", + name = "parser_builder", srcs = [ "CelPolicyParserBuilder.java", ], @@ -126,18 +126,67 @@ java_library( ], ) +java_library( + name = "compiler", + srcs = [ + "CelPolicyCompiler.java", + ], + deps = [ + ":policy", + ":validation_exception", + "//common", + ], +) + +java_library( + name = "compiler_builder", + srcs = [ + "CelPolicyCompilerBuilder.java", + ], + deps = [ + ":compiler", + "@maven//:com_google_errorprone_error_prone_annotations", + ], +) + +java_library( + name = "compiler_impl", + srcs = [ + "CelPolicyCompilerImpl.java", + ], + deps = [ + ":compiled_rule", + ":compiler", + ":compiler_builder", + ":policy", + ":rule_composer", + ":source", + ":validation_exception", + ":value_string", + "//bundle:cel", + "//common", + "//common:compiler_common", + "//common/ast", + "//common/types", + "//common/types:type_providers", + "//optimizer", + "//optimizer:optimization_exception", + "//optimizer:optimizer_builder", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + java_library( name = "yaml_config_parser", srcs = [ "CelPolicyYamlConfigParser.java", ], - tags = [ - ], deps = [ + ":common_internal", ":config", ":config_parser", ":parser_context", - ":policy_common_internal", ":source", ":validation_exception", "//common/internal", @@ -169,6 +218,18 @@ java_library( ], ) +java_library( + name = "compiled_rule", + srcs = ["CelCompiledRule.java"], + deps = [ + "//:auto_value", + "//bundle:cel", + "//common", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + java_library( name = "parser_context", srcs = [ @@ -181,7 +242,28 @@ java_library( ) java_library( - name = "policy_common_internal", + name = "rule_composer", + srcs = ["RuleComposer.java"], + visibility = ["//visibility:private"], + deps = [ + ":compiled_rule", + "//:auto_value", + "//bundle:cel", + "//common", + "//common:compiler_common", + "//common:mutable_ast", + "//common/ast", + "//extensions:optional_library", + "//optimizer:ast_optimizer", + "//optimizer:mutable_ast", + "//parser:operator", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + +java_library( + name = "common_internal", srcs = [ "YamlHelper.java", "YamlParserContextImpl.java", diff --git a/policy/src/main/java/dev/cel/policy/CelCompiledRule.java b/policy/src/main/java/dev/cel/policy/CelCompiledRule.java new file mode 100644 index 000000000..f7ee8340d --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelCompiledRule.java @@ -0,0 +1,83 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import com.google.auto.value.AutoOneOf; +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import dev.cel.bundle.Cel; +import dev.cel.common.CelAbstractSyntaxTree; + +/** Abstract representation of a compiled rule. */ +@AutoValue +abstract class CelCompiledRule { + abstract ImmutableList variables(); + + abstract ImmutableList matches(); + + abstract Cel cel(); + + @AutoValue + abstract static class CelCompiledVariable { + abstract String name(); + + abstract CelAbstractSyntaxTree ast(); + + static CelCompiledVariable create(String name, CelAbstractSyntaxTree ast) { + return new AutoValue_CelCompiledRule_CelCompiledVariable(name, ast); + } + } + + @AutoValue + abstract static class CelCompiledMatch { + abstract CelAbstractSyntaxTree condition(); + + abstract Result result(); + + @AutoOneOf(CelCompiledMatch.Result.Kind.class) + abstract static class Result { + abstract CelAbstractSyntaxTree output(); + + abstract CelCompiledRule rule(); + + abstract Kind kind(); + + static Result ofOutput(CelAbstractSyntaxTree value) { + return AutoOneOf_CelCompiledRule_CelCompiledMatch_Result.output(value); + } + + static Result ofRule(CelCompiledRule value) { + return AutoOneOf_CelCompiledRule_CelCompiledMatch_Result.rule(value); + } + + enum Kind { + OUTPUT, + RULE + } + } + + static CelCompiledMatch create( + CelAbstractSyntaxTree condition, CelCompiledMatch.Result result) { + return new AutoValue_CelCompiledRule_CelCompiledMatch(condition, result); + } + } + + static CelCompiledRule create( + ImmutableList variables, + ImmutableList matches, + Cel cel) { + return new AutoValue_CelCompiledRule(variables, matches, cel); + } +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java new file mode 100644 index 000000000..6a63f0389 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java @@ -0,0 +1,27 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import dev.cel.common.CelAbstractSyntaxTree; + +/** Public interface for compiling CEL policies. */ +public interface CelPolicyCompiler { + + /** + * Generates a single CEL AST from a collection of policy expressions associated with a CEL + * environment. + */ + CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidationException; +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerBuilder.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerBuilder.java new file mode 100644 index 000000000..a56054e03 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerBuilder.java @@ -0,0 +1,28 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import com.google.errorprone.annotations.CheckReturnValue; + +/** Interface for building an instance of {@link CelPolicyCompiler} */ +public interface CelPolicyCompilerBuilder { + + @CanIgnoreReturnValue + CelPolicyCompilerBuilder setVariablesPrefix(String prefix); + + @CheckReturnValue + CelPolicyCompiler build(); +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java new file mode 100644 index 000000000..4fb81f42b --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java @@ -0,0 +1,215 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.ImmutableList.toImmutableList; + +import com.google.common.base.Joiner; +import com.google.common.collect.ImmutableList; +import com.google.errorprone.annotations.CanIgnoreReturnValue; +import dev.cel.bundle.Cel; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelIssue; +import dev.cel.common.CelSource; +import dev.cel.common.CelSourceLocation; +import dev.cel.common.CelValidationException; +import dev.cel.common.CelVarDecl; +import dev.cel.common.ast.CelConstant; +import dev.cel.common.ast.CelExpr; +import dev.cel.common.types.CelType; +import dev.cel.common.types.SimpleType; +import dev.cel.optimizer.CelOptimizationException; +import dev.cel.optimizer.CelOptimizer; +import dev.cel.optimizer.CelOptimizerFactory; +import dev.cel.policy.CelCompiledRule.CelCompiledMatch; +import dev.cel.policy.CelCompiledRule.CelCompiledMatch.Result; +import dev.cel.policy.CelCompiledRule.CelCompiledVariable; +import dev.cel.policy.CelPolicy.Match; +import dev.cel.policy.CelPolicy.Variable; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** Package-private implementation for policy compiler. */ +final class CelPolicyCompilerImpl implements CelPolicyCompiler { + private static final String DEFAULT_VARIABLE_PREFIX = "variables."; + private final Cel cel; + private final String variablesPrefix; + + @Override + public CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidationException { + CompilerContext compilerContext = new CompilerContext(policy.policySource()); + CelCompiledRule compiledRule = compileRule(policy.rule(), cel, compilerContext); + if (compilerContext.hasError()) { + throw new CelPolicyValidationException(compilerContext.getIssueString()); + } + + CelOptimizer optimizer = + CelOptimizerFactory.standardCelOptimizerBuilder(compiledRule.cel()) + .addAstOptimizers( + RuleComposer.newInstance( + compiledRule, compilerContext.newVariableDeclarations, variablesPrefix)) + .build(); + + CelAbstractSyntaxTree ast; + + try { + // This is a minimal expression used as a basis of stitching together all the rules into a + // single graph. + ast = cel.compile("true").getAst(); + ast = optimizer.optimize(ast); + } catch (CelValidationException | CelOptimizationException e) { + // TODO: Surface these errors better + throw new CelPolicyValidationException("Failed composing the rules", e); + } + + return ast; + } + + private CelCompiledRule compileRule( + CelPolicy.Rule rule, Cel ruleCel, CompilerContext compilerContext) { + ImmutableList.Builder variableBuilder = ImmutableList.builder(); + for (Variable variable : rule.variables()) { + ValueString expression = variable.expression(); + CelAbstractSyntaxTree varAst; + CelType outputType = SimpleType.DYN; + try { + varAst = ruleCel.compile(expression.value()).getAst(); + outputType = varAst.getResultType(); + } catch (CelValidationException e) { + compilerContext.addIssue(expression.id(), e.getErrors()); + // A sentinel AST representing an error is created to allow compiler checks to continue + varAst = newErrorAst(); + } + String variableName = variable.name().value(); + CelVarDecl newVariable = + CelVarDecl.newVarDeclaration(variablesPrefix + variableName, outputType); + compilerContext.addNewVarDecl(newVariable); + ruleCel = ruleCel.toCelBuilder().addVarDeclarations(newVariable).build(); + variableBuilder.add(CelCompiledVariable.create(variableName, varAst)); + } + + ImmutableList.Builder matchBuilder = ImmutableList.builder(); + for (Match match : rule.matches()) { + CelAbstractSyntaxTree conditionAst; + try { + conditionAst = ruleCel.compile(match.condition().value()).getAst(); + } catch (CelValidationException e) { + compilerContext.addIssue(match.condition().id(), e.getErrors()); + continue; + } + + Result matchResult; + switch (match.result().kind()) { + case OUTPUT: + CelAbstractSyntaxTree outputAst; + try { + outputAst = ruleCel.compile(match.result().output().value()).getAst(); + } catch (CelValidationException e) { + compilerContext.addIssue(match.result().output().id(), e.getErrors()); + continue; + } + + matchResult = Result.ofOutput(outputAst); + break; + case RULE: + CelCompiledRule nestedRule = compileRule(match.result().rule(), ruleCel, compilerContext); + matchResult = Result.ofRule(nestedRule); + break; + default: + throw new IllegalArgumentException("Unexpected kind: " + match.result().kind()); + } + + matchBuilder.add(CelCompiledMatch.create(conditionAst, matchResult)); + } + + return CelCompiledRule.create(variableBuilder.build(), matchBuilder.build(), cel); + } + + private static CelAbstractSyntaxTree newErrorAst() { + return CelAbstractSyntaxTree.newParsedAst( + CelExpr.ofConstant(0, CelConstant.ofValue("*error*")), CelSource.newBuilder().build()); + } + + private static final class CompilerContext { + private static final Joiner JOINER = Joiner.on('\n'); + private final ArrayList issues; + private final ArrayList newVariableDeclarations; + private final CelPolicySource celPolicySource; + + private void addIssue(long id, List issues) { + for (CelIssue issue : issues) { + // Compute relative source and add them into the issues set + int position = Optional.ofNullable(celPolicySource.getPositionsMap().get(id)).orElse(-1); + position += issue.getSourceLocation().getColumn(); + CelSourceLocation loc = + celPolicySource.getOffsetLocation(position).orElse(CelSourceLocation.NONE); + this.issues.add(CelIssue.formatError(loc, issue.getMessage())); + } + } + + private void addNewVarDecl(CelVarDecl newVarDecl) { + newVariableDeclarations.add(newVarDecl); + } + + private boolean hasError() { + return !issues.isEmpty(); + } + + private String getIssueString() { + return JOINER.join( + issues.stream() + .map(iss -> iss.toDisplayString(celPolicySource)) + .collect(toImmutableList())); + } + + private CompilerContext(CelPolicySource celPolicySource) { + this.issues = new ArrayList<>(); + this.newVariableDeclarations = new ArrayList<>(); + this.celPolicySource = celPolicySource; + } + } + + static final class Builder implements CelPolicyCompilerBuilder { + private final Cel cel; + private String variablesPrefix; + + private Builder(Cel cel) { + this.cel = cel; + } + + @Override + @CanIgnoreReturnValue + public Builder setVariablesPrefix(String prefix) { + this.variablesPrefix = checkNotNull(prefix); + return this; + } + + @Override + public CelPolicyCompiler build() { + return new CelPolicyCompilerImpl(cel, this.variablesPrefix); + } + } + + static Builder newBuilder(Cel cel) { + return new Builder(cel).setVariablesPrefix(DEFAULT_VARIABLE_PREFIX); + } + + private CelPolicyCompilerImpl(Cel cel, String variablesPrefix) { + this.cel = checkNotNull(cel); + this.variablesPrefix = checkNotNull(variablesPrefix); + } +} diff --git a/policy/src/main/java/dev/cel/policy/RuleComposer.java b/policy/src/main/java/dev/cel/policy/RuleComposer.java new file mode 100644 index 000000000..9650b04a7 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/RuleComposer.java @@ -0,0 +1,135 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.base.Preconditions.checkNotNull; + +import com.google.auto.value.AutoValue; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import dev.cel.bundle.Cel; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelMutableAst; +import dev.cel.common.CelVarDecl; +import dev.cel.common.ast.CelConstant.Kind; +import dev.cel.extensions.CelOptionalLibrary.Function; +import dev.cel.optimizer.AstMutator; +import dev.cel.optimizer.CelAstOptimizer; +import dev.cel.parser.Operator; +import dev.cel.policy.CelCompiledRule.CelCompiledMatch; +import dev.cel.policy.CelCompiledRule.CelCompiledVariable; +import java.util.List; + +/** Package-private class for composing various rules into a single expression using optimizer. */ +final class RuleComposer implements CelAstOptimizer { + private static final int AST_MUTATOR_ITERATION_LIMIT = 1000; + private final CelCompiledRule compiledRule; + private final ImmutableList newVarDecls; + private final String variablePrefix; + private final AstMutator astMutator; + + @Override + public OptimizationResult optimize(CelAbstractSyntaxTree ast, Cel cel) { + RuleOptimizationResult result = optimizeRule(compiledRule); + return OptimizationResult.create(result.ast().toParsedAst(), newVarDecls, ImmutableList.of()); + } + + @AutoValue + abstract static class RuleOptimizationResult { + abstract CelMutableAst ast(); + + abstract Boolean isOptionalResult(); + + static RuleOptimizationResult create(CelMutableAst ast, boolean isOptionalResult) { + return new AutoValue_RuleComposer_RuleOptimizationResult(ast, isOptionalResult); + } + } + + private RuleOptimizationResult optimizeRule(CelCompiledRule compiledRule) { + CelMutableAst matchAst = astMutator.newGlobalCall(Function.OPTIONAL_NONE.getFunction()); + boolean isOptionalResult = true; + for (CelCompiledMatch match : Lists.reverse(compiledRule.matches())) { + CelAbstractSyntaxTree conditionAst = match.condition(); + boolean isTriviallyTrue = + conditionAst.getExpr().constantOrDefault().getKind().equals(Kind.BOOLEAN_VALUE) + && conditionAst.getExpr().constant().booleanValue(); + switch (match.result().kind()) { + case OUTPUT: + CelMutableAst outAst = CelMutableAst.fromCelAst(match.result().output()); + if (isTriviallyTrue) { + matchAst = outAst; + isOptionalResult = false; + continue; + } + if (isOptionalResult) { + outAst = astMutator.newGlobalCall(Function.OPTIONAL_OF.getFunction(), outAst); + } + + matchAst = + astMutator.newGlobalCall( + Operator.CONDITIONAL.getFunction(), + CelMutableAst.fromCelAst(conditionAst), + outAst, + matchAst); + continue; + case RULE: + RuleOptimizationResult nestedRule = optimizeRule(match.result().rule()); + CelMutableAst nestedRuleAst = nestedRule.ast(); + if (isOptionalResult && !nestedRule.isOptionalResult()) { + nestedRuleAst = + astMutator.newGlobalCall(Function.OPTIONAL_OF.getFunction(), nestedRuleAst); + } + if (!isOptionalResult && nestedRule.isOptionalResult()) { + matchAst = astMutator.newGlobalCall(Function.OPTIONAL_OF.getFunction(), matchAst); + isOptionalResult = true; + } + if (!isOptionalResult && !nestedRule.isOptionalResult()) { + throw new IllegalArgumentException("Subrule early terminates policy"); + } + matchAst = astMutator.newMemberCall(nestedRuleAst, Function.OR.getFunction(), matchAst); + break; + } + } + + CelMutableAst result = matchAst; + for (CelCompiledVariable variable : Lists.reverse(compiledRule.variables())) { + result = + astMutator.replaceSubtreeWithNewBindMacro( + result, + variablePrefix + variable.name(), + CelMutableAst.fromCelAst(variable.ast()), + result.expr(), + result.expr().id(), + true); + } + + result = astMutator.renumberIdsConsecutively(result); + + return RuleOptimizationResult.create(result, isOptionalResult); + } + + static RuleComposer newInstance( + CelCompiledRule compiledRule, List newVarDecls, String variablePrefix) { + return new RuleComposer(compiledRule, newVarDecls, variablePrefix); + } + + private RuleComposer( + CelCompiledRule compiledRule, List newVarDecls, String variablePrefix) { + this.compiledRule = checkNotNull(compiledRule); + this.newVarDecls = ImmutableList.copyOf(checkNotNull(newVarDecls)); + this.variablePrefix = variablePrefix; + this.astMutator = AstMutator.newInstance(AST_MUTATOR_ITERATION_LIMIT); + } +} diff --git a/policy/src/test/java/dev/cel/policy/BUILD.bazel b/policy/src/test/java/dev/cel/policy/BUILD.bazel index 38e88e5f4..1d863c77e 100644 --- a/policy/src/test/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/test/java/dev/cel/policy/BUILD.bazel @@ -12,9 +12,14 @@ java_library( deps = [ "//:java_truth", "//bundle:cel", + "//common", "//common:options", "//common/internal", + "//extensions:optional_library", + "//parser:macro", + "//parser:unparser", "//policy", + "//policy:compiler_impl", "//policy:config", "//policy:config_parser", "//policy:parser", @@ -22,10 +27,12 @@ java_library( "//policy:validation_exception", "//policy:yaml_config_parser", "//policy:yaml_parser", + "//runtime", "@maven//:com_google_api_grpc_proto_google_common_protos", "@maven//:com_google_guava_guava", "@maven//:com_google_testparameterinjector_test_parameter_injector", "@maven//:junit_junit", + "@maven//:org_yaml_snakeyaml", ], ) diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java new file mode 100644 index 000000000..f835e90fe --- /dev/null +++ b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java @@ -0,0 +1,237 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.truth.Truth.assertThat; +import static dev.cel.policy.PolicyTestHelper.readFromYaml; +import static org.junit.Assert.assertThrows; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.testing.junit.testparameterinjector.TestParameter; +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import com.google.testing.junit.testparameterinjector.TestParameterValue; +import com.google.testing.junit.testparameterinjector.TestParameterValuesProvider; +import dev.cel.bundle.Cel; +import dev.cel.bundle.CelFactory; +import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelOptions; +import dev.cel.extensions.CelOptionalLibrary; +import dev.cel.parser.CelStandardMacro; +import dev.cel.parser.CelUnparserFactory; +import dev.cel.policy.PolicyTestHelper.PolicyTestSuite; +import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection; +import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection.PolicyTestCase; +import dev.cel.policy.PolicyTestHelper.TestYamlPolicy; +import dev.cel.runtime.CelRuntime.CelFunctionBinding; +import java.util.Map.Entry; +import java.util.Optional; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public final class CelPolicyCompilerImplTest { + + private static final CelPolicyParser POLICY_PARSER = CelPolicyYamlParser.newBuilder().build(); + private static final CelPolicyConfigParser POLICY_CONFIG_PARSER = + CelPolicyYamlConfigParser.newInstance(); + + private static final CelOptions CEL_OPTIONS = + CelOptions.current().populateMacroCalls(true).build(); + + @Test + public void compileYamlPolicy_success(@TestParameter TestYamlPolicy yamlPolicy) throws Exception { + // Read config and produce an environment to compile policies + String configSource = yamlPolicy.readConfigYamlContent(); + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(configSource); + Cel cel = policyConfig.extend(newCel(), CEL_OPTIONS); + // Read the policy source + String policySource = yamlPolicy.readPolicyYamlContent(); + CelPolicy policy = POLICY_PARSER.parse(policySource); + + CelAbstractSyntaxTree ast = CelPolicyCompilerImpl.newBuilder(cel).build().compile(policy); + + assertThat(CelUnparserFactory.newUnparser().unparse(ast)).isEqualTo(yamlPolicy.getUnparsed()); + } + + @Test + public void compileYamlPolicy_containsError_throws() throws Exception { + // Read config and produce an environment to compile policies + String configSource = readFromYaml("errors/config.yaml"); + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(configSource); + Cel cel = policyConfig.extend(newCel(), CEL_OPTIONS); + // Read the policy source + String policyFilePath = "errors/policy.yaml"; + String policySource = readFromYaml(policyFilePath); + CelPolicy policy = POLICY_PARSER.parse(policySource, policyFilePath); + + CelPolicyValidationException e = + assertThrows( + CelPolicyValidationException.class, + () -> CelPolicyCompilerImpl.newBuilder(cel).build().compile(policy)); + + assertThat(e) + .hasMessageThat() + .isEqualTo( + "ERROR: errors/policy.yaml:19:19: undeclared reference to 'spec' (in container '')\n" + + " | expression: spec.labels\n" + + " | ..................^\n" + + "ERROR: errors/policy.yaml:21:50: mismatched input 'resource' expecting {'=='," + + " '!=', 'in', '<', '<=', '>=', '>', '&&', '||', '[', '(', ')', '.', '-', '?'," + + " '+', '*', '/', '%%'}\n" + + " | expression: variables.want.filter(l, !(lin resource.labels))\n" + + " | .................................................^\n" + + "ERROR: errors/policy.yaml:21:66: extraneous input ')' expecting \n" + + " | expression: variables.want.filter(l, !(lin resource.labels))\n" + + " | .................................................................^\n" + + "ERROR: errors/policy.yaml:23:27: mismatched input '2' expecting {'}', ','}\n" + + " | expression: \"{1:305 2:569}\"\n" + + " | ..........................^\n" + + "ERROR: errors/policy.yaml:31:65: extraneous input ']' expecting ')'\n" + + " | \"missing one or more required labels:" + + " %s\".format(variables.missing])\n" + + " | ................................................................^\n" + + "ERROR: errors/policy.yaml:34:57: undeclared reference to 'format' (in container" + + " '')\n" + + " | \"invalid values provided on one or more labels:" + + " %s\".format([variables.invalid])\n" + + " | ........................................................^\n" + + "ERROR: errors/policy.yaml:35:24: found no matching overload for '_==_' applied" + + " to '(bool, string)' (candidates: (%A0, %A0))\n" + + " | - condition: false == \"0\"\n" + + " | .......................^"); + } + + @Test + @SuppressWarnings("unchecked") + public void evaluateYamlPolicy_withCanonicalTestData( + @TestParameter(valuesProvider = EvaluablePolicyTestDataProvider.class) + EvaluablePolicyTestData testData) + throws Exception { + // Setup + // Read config and produce an environment to compile policies + String configSource = testData.yamlPolicy.readConfigYamlContent(); + CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(configSource); + Cel cel = policyConfig.extend(newCel(), CEL_OPTIONS); + // Read the policy source + String policySource = testData.yamlPolicy.readPolicyYamlContent(); + CelPolicy policy = POLICY_PARSER.parse(policySource); + CelAbstractSyntaxTree expectedOutputAst = cel.compile(testData.testCase.getOutput()).getAst(); + Object expectedOutput = cel.createProgram(expectedOutputAst).eval(); + + // Act + // Compile then evaluate the policy + CelAbstractSyntaxTree compiledPolicyAst = + CelPolicyCompilerImpl.newBuilder(cel).build().compile(policy); + ImmutableMap input = + testData.testCase.getInput().entrySet().stream() + .collect(toImmutableMap(Entry::getKey, e -> e.getValue().getValue())); + Object evalResult = cel.createProgram(compiledPolicyAst).eval(input); + + // Assert + // Note that policies may either produce an optional or a non-optional result, + // if all the rules included nested ones can always produce a default result when none of the + // condition matches + if (testData.yamlPolicy.producesOptionalResult()) { + Optional policyOutput = (Optional) evalResult; + if (policyOutput.isPresent()) { + assertThat(policyOutput).hasValue(expectedOutput); + } else { + assertThat(policyOutput).isEmpty(); + } + } else { + assertThat(evalResult).isEqualTo(expectedOutput); + } + } + + @Test + @SuppressWarnings("unchecked") + public void evaluateYamlPolicy_nestedRuleProducesOptionalOutput() throws Exception { + Cel cel = newCel(); + String policySource = + "name: nested_rule_with_optional_result\n" + + "rule:\n" + + " match:\n" + + " - rule:\n" + + " match:\n" + + " - condition: 'true'\n" + + " output: 'optional.of(true)'\n"; + CelPolicy policy = POLICY_PARSER.parse(policySource); + CelAbstractSyntaxTree compiledPolicyAst = + CelPolicyCompilerImpl.newBuilder(cel).build().compile(policy); + + Optional evalResult = (Optional) cel.createProgram(compiledPolicyAst).eval(); + + // Result is Optional> + assertThat(evalResult).hasValue(Optional.of(true)); + } + + private static final class EvaluablePolicyTestData { + private final TestYamlPolicy yamlPolicy; + private final PolicyTestCase testCase; + + private EvaluablePolicyTestData(TestYamlPolicy yamlPolicy, PolicyTestCase testCase) { + this.yamlPolicy = yamlPolicy; + this.testCase = testCase; + } + } + + private static final class EvaluablePolicyTestDataProvider extends TestParameterValuesProvider { + + @Override + protected ImmutableList provideValues(Context context) throws Exception { + ImmutableList.Builder builder = ImmutableList.builder(); + for (TestYamlPolicy yamlPolicy : TestYamlPolicy.values()) { + PolicyTestSuite testSuite = yamlPolicy.readTestYamlContent(); + for (PolicyTestSection testSection : testSuite.getSection()) { + for (PolicyTestCase testCase : testSection.getTests()) { + String testName = + String.format( + "%s %s %s", + yamlPolicy.getPolicyName(), testSection.getName(), testCase.getName()); + builder.add( + value(new EvaluablePolicyTestData(yamlPolicy, testCase)).withName(testName)); + } + } + } + + return builder.build(); + } + } + + private static Cel newCel() { + return CelFactory.standardCelBuilder() + .setStandardMacros(CelStandardMacro.STANDARD_MACROS) + .addCompilerLibraries(CelOptionalLibrary.INSTANCE) + .addRuntimeLibraries(CelOptionalLibrary.INSTANCE) + .setOptions(CEL_OPTIONS) + .addFunctionBindings( + CelFunctionBinding.from( + "locationCode_string", + String.class, + (ip) -> { + switch (ip) { + case "10.0.0.1": + return "us"; + case "10.0.0.2": + return "de"; + default: + return "ir"; + } + })) + .build(); + } +} diff --git a/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java b/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java index c04d15440..ed1b94a66 100644 --- a/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java +++ b/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java @@ -16,10 +16,16 @@ import static java.nio.charset.StandardCharsets.UTF_8; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; import com.google.common.io.Resources; import java.io.IOException; import java.net.URL; +import java.util.List; +import java.util.Map; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; /** Package-private class to assist with policy testing. */ final class PolicyTestHelper { @@ -87,12 +93,126 @@ String getUnparsed() { String readPolicyYamlContent() throws IOException { return readFromYaml(String.format("%s/policy.yaml", name)); } + + String readConfigYamlContent() throws IOException { + return readFromYaml(String.format("%s/config.yaml", name)); + } + + PolicyTestSuite readTestYamlContent() throws IOException { + Yaml yaml = new Yaml(new Constructor(PolicyTestSuite.class, new LoaderOptions())); + String testContent = readFile(String.format("%s/tests.yaml", name)); + + return yaml.load(testContent); + } } static String readFromYaml(String yamlPath) throws IOException { return readFile(yamlPath); } + /** + * TestSuite describes a set of tests divided by section. + * + *

Visibility must be public for YAML deserialization to work. This is effectively + * package-private since the outer class is. + */ + @VisibleForTesting + public static final class PolicyTestSuite { + private String description; + private List section; + + public void setDescription(String description) { + this.description = description; + } + + public void setSection(List section) { + this.section = section; + } + + public String getDescription() { + return description; + } + + public List getSection() { + return section; + } + + @VisibleForTesting + public static final class PolicyTestSection { + private String name; + private List tests; + + public void setName(String name) { + this.name = name; + } + + public void setTests(List tests) { + this.tests = tests; + } + + public String getName() { + return name; + } + + public List getTests() { + return tests; + } + + @VisibleForTesting + public static final class PolicyTestCase { + private String name; + private Map input; + private String output; + + public void setName(String name) { + this.name = name; + } + + public void setInput(Map input) { + this.input = input; + } + + public void setOutput(String output) { + this.output = output; + } + + public String getName() { + return name; + } + + public Map getInput() { + return input; + } + + public String getOutput() { + return output; + } + + @VisibleForTesting + public static final class PolicyTestInput { + private Object value; + private String expr; + + public Object getValue() { + return value; + } + + public void setValue(Object value) { + this.value = value; + } + + public String getExpr() { + return expr; + } + + public void setExpr(String expr) { + this.expr = expr; + } + } + } + } + } + private static URL getResource(String path) { return Resources.getResource(Ascii.toLowerCase(path)); } diff --git a/policy/src/test/resources/errors/policy.yaml b/policy/src/test/resources/errors/policy.yaml index 338ba3995..e322e8a83 100644 --- a/policy/src/test/resources/errors/policy.yaml +++ b/policy/src/test/resources/errors/policy.yaml @@ -32,3 +32,6 @@ rule: - condition: variables.invalid.size() > 0 output: | "invalid values provided on one or more labels: %s".format([variables.invalid]) + - condition: false == "0" + output: | + "wrong type" From 29f344b4348c6611ce7aa8a1976d0800649693fe Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Wed, 3 Jul 2024 13:27:05 -0700 Subject: [PATCH 21/33] Add factories for policy parser and policy compiler PiperOrigin-RevId: 649181111 --- policy/BUILD.bazel | 16 ++----- .../src/main/java/dev/cel/policy/BUILD.bazel | 27 +++++++++++ .../cel/policy/CelPolicyCompilerFactory.java | 46 ++++++++++++++++++ .../cel/policy/CelPolicyParserFactory.java | 39 +++++++++++++++ .../src/test/java/dev/cel/policy/BUILD.bazel | 7 +-- .../policy/CelPolicyCompilerFactoryTest.java | 47 +++++++++++++++++++ .../cel/policy/CelPolicyCompilerImplTest.java | 15 +++--- .../policy/CelPolicyYamlConfigParserTest.java | 2 +- .../cel/policy/CelPolicyYamlParserTest.java | 3 +- 9 files changed, 178 insertions(+), 24 deletions(-) create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyCompilerFactory.java create mode 100644 policy/src/main/java/dev/cel/policy/CelPolicyParserFactory.java create mode 100644 policy/src/test/java/dev/cel/policy/CelPolicyCompilerFactoryTest.java diff --git a/policy/BUILD.bazel b/policy/BUILD.bazel index 2781b5663..ab45214bf 100644 --- a/policy/BUILD.bazel +++ b/policy/BUILD.bazel @@ -34,19 +34,11 @@ java_library( ) java_library( - name = "compiler_impl", - visibility = ["//visibility:public"], - exports = ["//policy/src/main/java/dev/cel/policy:compiler_impl"], + name = "parser_factory", + exports = ["//policy/src/main/java/dev/cel/policy:parser_factory"], ) java_library( - name = "yaml_parser", - visibility = ["//visibility:public"], - exports = ["//policy/src/main/java/dev/cel/policy:yaml_parser"], -) - -java_library( - name = "yaml_config_parser", - visibility = ["//visibility:public"], - exports = ["//policy/src/main/java/dev/cel/policy:yaml_config_parser"], + name = "compiler_factory", + exports = ["//policy/src/main/java/dev/cel/policy:compiler_factory"], ) diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index bf755f2e9..3ac84d7e4 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -83,6 +83,18 @@ java_library( ], ) +java_library( + name = "parser_factory", + srcs = ["CelPolicyParserFactory.java"], + deps = [ + ":config_parser", + ":parser_builder", + ":yaml_config_parser", + ":yaml_parser", + "@maven//:org_yaml_snakeyaml", + ], +) + java_library( name = "yaml_parser", srcs = [ @@ -177,6 +189,21 @@ java_library( ], ) +java_library( + name = "compiler_factory", + srcs = ["CelPolicyCompilerFactory.java"], + deps = [ + ":compiler_builder", + ":compiler_impl", + "//bundle:cel", + "//checker:checker_builder", + "//compiler", + "//compiler:compiler_builder", + "//parser:parser_builder", + "//runtime", + ], +) + java_library( name = "yaml_config_parser", srcs = [ diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerFactory.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerFactory.java new file mode 100644 index 000000000..8641e5bb7 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerFactory.java @@ -0,0 +1,46 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import dev.cel.bundle.Cel; +import dev.cel.bundle.CelFactory; +import dev.cel.checker.CelChecker; +import dev.cel.compiler.CelCompiler; +import dev.cel.compiler.CelCompilerFactory; +import dev.cel.parser.CelParser; +import dev.cel.runtime.CelRuntime; + +/** Factory class for producing policy compilers. */ +public final class CelPolicyCompilerFactory { + + /** Create a builder for constructing a {@link CelPolicyCompiler} instance. */ + public static CelPolicyCompilerBuilder newPolicyCompiler(Cel cel) { + return CelPolicyCompilerImpl.newBuilder(cel); + } + + /** Create a builder for constructing a {@link CelPolicyCompiler} instance. */ + public static CelPolicyCompilerBuilder newPolicyCompiler( + CelCompiler celCompiler, CelRuntime celRuntime) { + return newPolicyCompiler(CelFactory.combine(celCompiler, celRuntime)); + } + + /** Create a builder for constructing a {@link CelPolicyCompiler} instance. */ + public static CelPolicyCompilerBuilder newPolicyCompiler( + CelParser celParser, CelChecker celChecker, CelRuntime celRuntime) { + return newPolicyCompiler(CelCompilerFactory.combine(celParser, celChecker), celRuntime); + } + + private CelPolicyCompilerFactory() {} +} diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyParserFactory.java b/policy/src/main/java/dev/cel/policy/CelPolicyParserFactory.java new file mode 100644 index 000000000..3897bef41 --- /dev/null +++ b/policy/src/main/java/dev/cel/policy/CelPolicyParserFactory.java @@ -0,0 +1,39 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import org.yaml.snakeyaml.nodes.Node; + +/** Factory class for producing policy parser and policy config parsers. */ +public final class CelPolicyParserFactory { + + /** + * Configure a builder to construct a {@link CelPolicyParser} instance that takes in a YAML + * document. + */ + public static CelPolicyParserBuilder newYamlParserBuilder() { + return CelPolicyYamlParser.newBuilder(); + } + + /** + * Configure a builder to construct a {@link CelPolicyConfigParser} instance that takes in a YAML + * document. + */ + public static CelPolicyConfigParser newYamlConfigParser() { + return CelPolicyYamlConfigParser.newInstance(); + } + + private CelPolicyParserFactory() {} +} diff --git a/policy/src/test/java/dev/cel/policy/BUILD.bazel b/policy/src/test/java/dev/cel/policy/BUILD.bazel index 1d863c77e..05154b7e6 100644 --- a/policy/src/test/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/test/java/dev/cel/policy/BUILD.bazel @@ -15,18 +15,19 @@ java_library( "//common", "//common:options", "//common/internal", + "//compiler", "//extensions:optional_library", + "//parser", "//parser:macro", "//parser:unparser", "//policy", - "//policy:compiler_impl", + "//policy:compiler_factory", "//policy:config", "//policy:config_parser", "//policy:parser", + "//policy:parser_factory", "//policy:source", "//policy:validation_exception", - "//policy:yaml_config_parser", - "//policy:yaml_parser", "//runtime", "@maven//:com_google_api_grpc_proto_google_common_protos", "@maven//:com_google_guava_guava", diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerFactoryTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerFactoryTest.java new file mode 100644 index 000000000..1f323256b --- /dev/null +++ b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerFactoryTest.java @@ -0,0 +1,47 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dev.cel.policy; + +import static com.google.common.truth.Truth.assertThat; + +import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.compiler.CelCompilerFactory; +import dev.cel.parser.CelParserFactory; +import dev.cel.runtime.CelRuntimeFactory; +import org.junit.Test; +import org.junit.runner.RunWith; + +@RunWith(TestParameterInjector.class) +public final class CelPolicyCompilerFactoryTest { + + @Test + public void newPolicyCompiler_compilerRuntimeCombined() { + assertThat( + CelPolicyCompilerFactory.newPolicyCompiler( + CelCompilerFactory.standardCelCompilerBuilder().build(), + CelRuntimeFactory.standardCelRuntimeBuilder().build())) + .isNotNull(); + } + + @Test + public void newPolicyCompiler_parserCheckerRuntimeCombined() { + assertThat( + CelPolicyCompilerFactory.newPolicyCompiler( + CelParserFactory.standardCelParserBuilder().build(), + CelCompilerFactory.standardCelCheckerBuilder().build(), + CelRuntimeFactory.standardCelRuntimeBuilder().build())) + .isNotNull(); + } +} diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java index f835e90fe..41cefcae1 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java @@ -45,10 +45,10 @@ @RunWith(TestParameterInjector.class) public final class CelPolicyCompilerImplTest { - private static final CelPolicyParser POLICY_PARSER = CelPolicyYamlParser.newBuilder().build(); + private static final CelPolicyParser POLICY_PARSER = + CelPolicyParserFactory.newYamlParserBuilder().build(); private static final CelPolicyConfigParser POLICY_CONFIG_PARSER = - CelPolicyYamlConfigParser.newInstance(); - + CelPolicyParserFactory.newYamlConfigParser(); private static final CelOptions CEL_OPTIONS = CelOptions.current().populateMacroCalls(true).build(); @@ -62,7 +62,8 @@ public void compileYamlPolicy_success(@TestParameter TestYamlPolicy yamlPolicy) String policySource = yamlPolicy.readPolicyYamlContent(); CelPolicy policy = POLICY_PARSER.parse(policySource); - CelAbstractSyntaxTree ast = CelPolicyCompilerImpl.newBuilder(cel).build().compile(policy); + CelAbstractSyntaxTree ast = + CelPolicyCompilerFactory.newPolicyCompiler(cel).build().compile(policy); assertThat(CelUnparserFactory.newUnparser().unparse(ast)).isEqualTo(yamlPolicy.getUnparsed()); } @@ -81,7 +82,7 @@ public void compileYamlPolicy_containsError_throws() throws Exception { CelPolicyValidationException e = assertThrows( CelPolicyValidationException.class, - () -> CelPolicyCompilerImpl.newBuilder(cel).build().compile(policy)); + () -> CelPolicyCompilerFactory.newPolicyCompiler(cel).build().compile(policy)); assertThat(e) .hasMessageThat() @@ -135,7 +136,7 @@ public void evaluateYamlPolicy_withCanonicalTestData( // Act // Compile then evaluate the policy CelAbstractSyntaxTree compiledPolicyAst = - CelPolicyCompilerImpl.newBuilder(cel).build().compile(policy); + CelPolicyCompilerFactory.newPolicyCompiler(cel).build().compile(policy); ImmutableMap input = testData.testCase.getInput().entrySet().stream() .collect(toImmutableMap(Entry::getKey, e -> e.getValue().getValue())); @@ -171,7 +172,7 @@ public void evaluateYamlPolicy_nestedRuleProducesOptionalOutput() throws Excepti + " output: 'optional.of(true)'\n"; CelPolicy policy = POLICY_PARSER.parse(policySource); CelAbstractSyntaxTree compiledPolicyAst = - CelPolicyCompilerImpl.newBuilder(cel).build().compile(policy); + CelPolicyCompilerFactory.newPolicyCompiler(cel).build().compile(policy); Optional evalResult = (Optional) cel.createProgram(compiledPolicyAst).eval(); diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java index 235126270..6da658729 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyYamlConfigParserTest.java @@ -41,7 +41,7 @@ public final class CelPolicyYamlConfigParserTest { .build(); private static final CelPolicyConfigParser POLICY_CONFIG_PARSER = - CelPolicyYamlConfigParser.newInstance(); + CelPolicyParserFactory.newYamlConfigParser(); @Test public void config_setBasicProperties() throws Exception { diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java index fc19d8f0c..6a7cfc800 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java @@ -26,7 +26,8 @@ @RunWith(TestParameterInjector.class) public final class CelPolicyYamlParserTest { - private static final CelPolicyParser POLICY_PARSER = CelPolicyYamlParser.newBuilder().build(); + private static final CelPolicyParser POLICY_PARSER = + CelPolicyParserFactory.newYamlParserBuilder().build(); @Test public void parseYamlPolicy_success(@TestParameter TestYamlPolicy yamlPolicy) throws Exception { From 20c1933c7deb061b949536d48cd7911a54e6cc69 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Wed, 3 Jul 2024 13:36:37 -0700 Subject: [PATCH 22/33] Add capability to visit custom tags PiperOrigin-RevId: 649184086 --- policy/BUILD.bazel | 10 ++ .../src/main/java/dev/cel/policy/BUILD.bazel | 5 +- .../main/java/dev/cel/policy/CelPolicy.java | 24 ++- .../java/dev/cel/policy/CelPolicyParser.java | 31 ++-- .../dev/cel/policy/CelPolicyYamlParser.java | 160 ++++++++++++------ .../java/dev/cel/policy/ParserContext.java | 33 +++- .../main/java/dev/cel/policy/ValueString.java | 10 +- .../main/java/dev/cel/policy/YamlHelper.java | 14 +- .../dev/cel/policy/YamlParserContextImpl.java | 16 ++ .../src/test/java/dev/cel/policy/BUILD.bazel | 2 + .../cel/policy/CelPolicyCompilerImplTest.java | 3 +- .../cel/policy/CelPolicyYamlParserTest.java | 3 +- .../java/dev/cel/policy/PolicyTestHelper.java | 120 ++++++++++++- policy/src/test/resources/k8s/config.yaml | 33 ++++ policy/src/test/resources/k8s/policy.yaml | 36 ++++ policy/src/test/resources/k8s/tests.yaml | 31 ++++ 16 files changed, 443 insertions(+), 88 deletions(-) create mode 100644 policy/src/test/resources/k8s/config.yaml create mode 100644 policy/src/test/resources/k8s/policy.yaml create mode 100644 policy/src/test/resources/k8s/tests.yaml diff --git a/policy/BUILD.bazel b/policy/BUILD.bazel index ab45214bf..222619d9f 100644 --- a/policy/BUILD.bazel +++ b/policy/BUILD.bazel @@ -8,6 +8,11 @@ java_library( exports = ["//policy/src/main/java/dev/cel/policy"], ) +java_library( + name = "value_string", + exports = ["//policy/src/main/java/dev/cel/policy:value_string"], +) + java_library( name = "source", exports = ["//policy/src/main/java/dev/cel/policy:source"], @@ -33,6 +38,11 @@ java_library( exports = ["//policy/src/main/java/dev/cel/policy:parser"], ) +java_library( + name = "parser_context", + exports = ["//policy/src/main/java/dev/cel/policy:parser_context"], +) + java_library( name = "parser_factory", exports = ["//policy/src/main/java/dev/cel/policy:parser_factory"], diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index 3ac84d7e4..407ce2f4e 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -109,6 +109,7 @@ java_library( ":source", ":validation_exception", ":value_string", + "//common:source", "//common/internal", "@maven//:com_google_guava_guava", "@maven//:org_yaml_snakeyaml", @@ -227,7 +228,6 @@ java_library( srcs = [ "ValueString.java", ], - visibility = ["//visibility:private"], deps = [ "//:auto_value", ], @@ -262,8 +262,9 @@ java_library( srcs = [ "ParserContext.java", ], - visibility = ["//visibility:private"], deps = [ + ":policy", + ":value_string", "//common:source", ], ) diff --git a/policy/src/main/java/dev/cel/policy/CelPolicy.java b/policy/src/main/java/dev/cel/policy/CelPolicy.java index f64d8cc58..0c14b745f 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicy.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicy.java @@ -19,9 +19,11 @@ import com.google.auto.value.AutoOneOf; import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.errorprone.annotations.CanIgnoreReturnValue; import java.util.Arrays; +import java.util.Map; import java.util.Optional; /** @@ -37,11 +39,14 @@ public abstract class CelPolicy { public abstract CelPolicySource policySource(); + public abstract ImmutableMap metadata(); + /** Creates a new builder to construct a {@link CelPolicy} instance. */ public static Builder newBuilder() { return new AutoValue_CelPolicy.Builder() .setName(ValueString.of(0, "")) - .setRule(Rule.newBuilder().build()); + .setRule(Rule.newBuilder().build()) + .setMetadata(ImmutableMap.of()); } /** Builder for {@link CelPolicy}. */ @@ -55,6 +60,23 @@ public abstract static class Builder { public abstract Builder setPolicySource(CelPolicySource policySource); + // This should stay package-private to encourage add/set methods to be used instead. + abstract ImmutableMap.Builder metadataBuilder(); + + public abstract Builder setMetadata(ImmutableMap value); + + @CanIgnoreReturnValue + public Builder putMetadata(String key, Object value) { + metadataBuilder().put(key, value); + return this; + } + + @CanIgnoreReturnValue + public Builder putMetadata(Map map) { + metadataBuilder().putAll(map); + return this; + } + public abstract CelPolicy build(); } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyParser.java index 315ece06c..cfeceb89b 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyParser.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyParser.java @@ -14,6 +14,8 @@ package dev.cel.policy; +import dev.cel.policy.ParserContext.PolicyParserContext; + /** CelPolicyParser is the interface for parsing policies into a canonical Policy representation. */ public interface CelPolicyParser { @@ -41,8 +43,12 @@ interface TagVisitor { * allow for continued parsing within a custom tag. */ default void visitPolicyTag( - ParserContext ctx, long id, String fieldName, T node, CelPolicy.Builder policyBuilder) { - ctx.reportError(id, String.format("Unsupported policy tag: %s", fieldName)); + PolicyParserContext ctx, + long id, + String tagName, + T node, + CelPolicy.Builder policyBuilder) { + ctx.reportError(id, String.format("Unsupported policy tag: %s", tagName)); } /** @@ -50,12 +56,13 @@ default void visitPolicyTag( * policy and current rule to allow for continued parsing within custom tags. */ default void visitRuleTag( - ParserContext ctx, + PolicyParserContext ctx, long id, - String fieldName, + String tagName, T node, + CelPolicy.Builder policyBuilder, CelPolicy.Rule.Builder ruleBuilder) { - ctx.reportError(id, String.format("Unsupported rule tag: %s", fieldName)); + ctx.reportError(id, String.format("Unsupported rule tag: %s", tagName)); } /** @@ -63,12 +70,13 @@ default void visitRuleTag( * policy and current match to allow for continued parsing within custom tags. */ default void visitMatchTag( - ParserContext ctx, + PolicyParserContext ctx, long id, - String fieldName, + String tagName, T node, + CelPolicy.Builder policyBuilder, CelPolicy.Match.Builder matchBuilder) { - ctx.reportError(id, String.format("Unsupported match tag: %s", fieldName)); + ctx.reportError(id, String.format("Unsupported match tag: %s", tagName)); } /** @@ -76,12 +84,13 @@ default void visitMatchTag( * parent policy and current variable to allow for continued parsing within custom tags. */ default void visitVariableTag( - ParserContext ctx, + PolicyParserContext ctx, long id, - String fieldName, + String tagName, T node, + CelPolicy.Builder policyBuilder, CelPolicy.Variable.Builder variableBuilder) { - ctx.reportError(id, String.format("Unsupported variable tag: %s", fieldName)); + ctx.reportError(id, String.format("Unsupported variable tag: %s", tagName)); } } } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java index 64921bee9..5d36185ab 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java @@ -15,16 +15,19 @@ package dev.cel.policy; import static com.google.common.base.Preconditions.checkNotNull; +import static dev.cel.policy.YamlHelper.ERROR; import static dev.cel.policy.YamlHelper.assertRequiredFields; import static dev.cel.policy.YamlHelper.assertYamlType; -import static dev.cel.policy.YamlHelper.newValueString; import com.google.common.collect.ImmutableSet; +import dev.cel.common.Source; import dev.cel.common.internal.CelCodePointArray; import dev.cel.policy.CelPolicy.Match; +import dev.cel.policy.CelPolicy.Match.Result; import dev.cel.policy.CelPolicy.Variable; +import dev.cel.policy.ParserContext.PolicyParserContext; import dev.cel.policy.YamlHelper.YamlNodeType; -import java.util.Optional; +import java.util.Map; import org.yaml.snakeyaml.nodes.MappingNode; import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.NodeTuple; @@ -33,6 +36,15 @@ final class CelPolicyYamlParser implements CelPolicyParser { + // Sentinel values for parsing errors + private static final Match ERROR_MATCH = + Match.newBuilder() + .setCondition(ValueString.newBuilder().setValue(ERROR).build()) + .setResult(Result.ofOutput(ValueString.newBuilder().setValue(ERROR).build())) + .build(); + private static final Variable ERROR_VARIABLE = + Variable.newBuilder().setName(ValueString.newBuilder().setValue(ERROR).build()).build(); + private final TagVisitor tagVisitor; @Override @@ -43,45 +55,42 @@ public CelPolicy parse(String policySource) throws CelPolicyValidationException @Override public CelPolicy parse(String policySource, String description) throws CelPolicyValidationException { - ParserImpl parser = new ParserImpl(tagVisitor); - return parser.parseYaml(policySource, description); + ParserImpl parser = new ParserImpl(tagVisitor, policySource, description); + return parser.parseYaml(); } - private static class ParserImpl { + private static class ParserImpl implements PolicyParserContext { private final TagVisitor tagVisitor; + private final CelCodePointArray policySource; + private final String description; + private final ParserContext ctx; - private CelPolicy parseYaml(String policySource, String description) - throws CelPolicyValidationException { + private CelPolicy parseYaml() throws CelPolicyValidationException { Node node; try { - node = YamlHelper.parseYamlSource(policySource); + node = YamlHelper.parseYamlSource(policySource.toString()); } catch (RuntimeException e) { throw new CelPolicyValidationException("YAML document is malformed: " + e.getMessage(), e); } - CelCodePointArray policyCodePoints = CelCodePointArray.fromString(policySource); - ParserContext ctx = YamlParserContextImpl.newInstance(policyCodePoints); - - CelPolicy.Builder policyBuilder = parsePolicy(ctx, node); - CelPolicySource celPolicySource = - CelPolicySource.newBuilder(policyCodePoints) - .setDescription(description) - .setPositionsMap(ctx.getIdToOffsetMap()) - .build(); + CelPolicy celPolicy = parsePolicy(this, node); if (ctx.hasError()) { - throw new CelPolicyValidationException(ctx.getIssueString(celPolicySource)); + throw new CelPolicyValidationException(ctx.getIssueString(celPolicy.policySource())); } - return policyBuilder.setPolicySource(celPolicySource).build(); + return celPolicy; } - private CelPolicy.Builder parsePolicy(ParserContext ctx, Node node) { + @Override + public CelPolicy parsePolicy(PolicyParserContext ctx, Node node) { CelPolicy.Builder policyBuilder = CelPolicy.newBuilder(); + CelPolicySource.Builder sourceBuilder = + CelPolicySource.newBuilder(policySource).setDescription(description); long id = ctx.collectMetadata(node); if (!assertYamlType(ctx, id, node, YamlNodeType.MAP)) { - return policyBuilder; + return policyBuilder.setPolicySource(sourceBuilder.build()).build(); } MappingNode rootNode = (MappingNode) node; @@ -96,10 +105,10 @@ private CelPolicy.Builder parsePolicy(ParserContext ctx, Node node) { String fieldName = ((ScalarNode) keyNode).getValue(); switch (fieldName) { case "name": - policyBuilder.setName(newValueString(ctx, valueNode)); + policyBuilder.setName(ctx.newValueString(valueNode)); break; case "rule": - policyBuilder.setRule(parseRule(ctx, valueNode)); + policyBuilder.setRule(parseRule(ctx, policyBuilder, valueNode)); break; default: tagVisitor.visitPolicyTag(ctx, keyId, fieldName, valueNode, policyBuilder); @@ -107,10 +116,14 @@ private CelPolicy.Builder parsePolicy(ParserContext ctx, Node node) { } } - return policyBuilder; + return policyBuilder + .setPolicySource(sourceBuilder.setPositionsMap(ctx.getIdToOffsetMap()).build()) + .build(); } - private CelPolicy.Rule parseRule(ParserContext ctx, Node node) { + @Override + public CelPolicy.Rule parseRule( + PolicyParserContext ctx, CelPolicy.Builder policyBuilder, Node node) { long valueId = ctx.collectMetadata(node); CelPolicy.Rule.Builder ruleBuilder = CelPolicy.Rule.newBuilder(); if (!assertYamlType(ctx, valueId, node, YamlNodeType.MAP)) { @@ -127,26 +140,27 @@ private CelPolicy.Rule parseRule(ParserContext ctx, Node node) { Node value = nodeTuple.getValueNode(); switch (fieldName) { case "id": - ruleBuilder.setId(newValueString(ctx, value)); + ruleBuilder.setId(ctx.newValueString(value)); break; case "description": - ruleBuilder.setDescription(newValueString(ctx, value)); + ruleBuilder.setDescription(ctx.newValueString(value)); break; case "variables": - ruleBuilder.addVariables(parseVariables(ctx, value)); + ruleBuilder.addVariables(parseVariables(ctx, policyBuilder, value)); break; case "match": - ruleBuilder.addMatches(parseMatches(ctx, value)); + ruleBuilder.addMatches(parseMatches(ctx, policyBuilder, value)); break; default: - tagVisitor.visitRuleTag(ctx, tagId, fieldName, value, ruleBuilder); + tagVisitor.visitRuleTag(ctx, tagId, fieldName, value, policyBuilder, ruleBuilder); break; } } return ruleBuilder.build(); } - private ImmutableSet parseMatches(ParserContext ctx, Node node) { + private ImmutableSet parseMatches( + PolicyParserContext ctx, CelPolicy.Builder policyBuilder, Node node) { long valueId = ctx.collectMetadata(node); ImmutableSet.Builder matchesBuilder = ImmutableSet.builder(); if (!assertYamlType(ctx, valueId, node, YamlNodeType.LIST)) { @@ -155,16 +169,18 @@ private ImmutableSet parseMatches(ParserContext ctx, Node SequenceNode matchListNode = (SequenceNode) node; for (Node elementNode : matchListNode.getValue()) { - parseMatch(ctx, elementNode).ifPresent(matchesBuilder::add); + matchesBuilder.add(parseMatch(ctx, policyBuilder, elementNode)); } return matchesBuilder.build(); } - private Optional parseMatch(ParserContext ctx, Node node) { + @Override + public CelPolicy.Match parseMatch( + PolicyParserContext ctx, CelPolicy.Builder policyBuilder, Node node) { long nodeId = ctx.collectMetadata(node); if (!assertYamlType(ctx, nodeId, node, YamlNodeType.MAP)) { - return Optional.empty(); + return ERROR_MATCH; } MappingNode matchNode = (MappingNode) node; CelPolicy.Match.Builder matchBuilder = @@ -179,7 +195,7 @@ private Optional parseMatch(ParserContext ctx, Node node) Node value = nodeTuple.getValueNode(); switch (fieldName) { case "condition": - matchBuilder.setCondition(newValueString(ctx, value)); + matchBuilder.setCondition(ctx.newValueString(value)); break; case "output": matchBuilder @@ -187,7 +203,7 @@ private Optional parseMatch(ParserContext ctx, Node node) .filter(result -> result.kind().equals(Match.Result.Kind.RULE)) .ifPresent( result -> ctx.reportError(tagId, "Only the rule or the output may be set")); - matchBuilder.setResult(Match.Result.ofOutput(newValueString(ctx, value))); + matchBuilder.setResult(Match.Result.ofOutput(ctx.newValueString(value))); break; case "rule": matchBuilder @@ -195,22 +211,23 @@ private Optional parseMatch(ParserContext ctx, Node node) .filter(result -> result.kind().equals(Match.Result.Kind.OUTPUT)) .ifPresent( result -> ctx.reportError(tagId, "Only the rule or the output may be set")); - matchBuilder.setResult(Match.Result.ofRule(parseRule(ctx, value))); + matchBuilder.setResult(Match.Result.ofRule(parseRule(ctx, policyBuilder, value))); break; default: - tagVisitor.visitMatchTag(ctx, tagId, fieldName, value, matchBuilder); + tagVisitor.visitMatchTag(ctx, tagId, fieldName, value, policyBuilder, matchBuilder); break; } } if (!assertRequiredFields(ctx, nodeId, matchBuilder.getMissingRequiredFieldNames())) { - return Optional.empty(); + return ERROR_MATCH; } - return Optional.of(matchBuilder.build()); + return matchBuilder.build(); } - private ImmutableSet parseVariables(ParserContext ctx, Node node) { + private ImmutableSet parseVariables( + PolicyParserContext ctx, CelPolicy.Builder policyBuilder, Node node) { long valueId = ctx.collectMetadata(node); ImmutableSet.Builder variableBuilder = ImmutableSet.builder(); if (!assertYamlType(ctx, valueId, node, YamlNodeType.LIST)) { @@ -219,17 +236,20 @@ private ImmutableSet parseVariables(ParserContext ctx, SequenceNode variableListNode = (SequenceNode) node; for (Node elementNode : variableListNode.getValue()) { - long id = ctx.collectMetadata(elementNode); - if (!assertYamlType(ctx, id, elementNode, YamlNodeType.MAP)) { - continue; - } - variableBuilder.add(parseVariable(ctx, (MappingNode) elementNode)); + variableBuilder.add(parseVariable(ctx, policyBuilder, elementNode)); } return variableBuilder.build(); } - private CelPolicy.Variable parseVariable(ParserContext ctx, MappingNode variableMap) { + @Override + public CelPolicy.Variable parseVariable( + PolicyParserContext ctx, CelPolicy.Builder policyBuilder, Node node) { + long id = ctx.collectMetadata(node); + if (!assertYamlType(ctx, id, node, YamlNodeType.MAP)) { + return ERROR_VARIABLE; + } + MappingNode variableMap = (MappingNode) node; Variable.Builder builder = Variable.newBuilder(); for (NodeTuple nodeTuple : variableMap.getValue()) { @@ -239,13 +259,13 @@ private CelPolicy.Variable parseVariable(ParserContext ctx, MappingNode va String keyName = ((ScalarNode) keyNode).getValue(); switch (keyName) { case "name": - builder.setName(newValueString(ctx, valueNode)); + builder.setName(ctx.newValueString(valueNode)); break; case "expression": - builder.setExpression(newValueString(ctx, valueNode)); + builder.setExpression(ctx.newValueString(valueNode)); break; default: - tagVisitor.visitVariableTag(ctx, keyId, keyName, valueNode, builder); + tagVisitor.visitVariableTag(ctx, keyId, keyName, valueNode, policyBuilder, builder); break; } } @@ -253,8 +273,46 @@ private CelPolicy.Variable parseVariable(ParserContext ctx, MappingNode va return builder.build(); } - private ParserImpl(TagVisitor tagVisitor) { + private ParserImpl(TagVisitor tagVisitor, String source, String description) { this.tagVisitor = tagVisitor; + this.policySource = CelCodePointArray.fromString(source); + this.description = description; + this.ctx = YamlParserContextImpl.newInstance(policySource); + } + + @Override + public long nextId() { + return ctx.nextId(); + } + + @Override + public long collectMetadata(Node node) { + return ctx.collectMetadata(node); + } + + @Override + public void reportError(long id, String message) { + ctx.reportError(id, message); + } + + @Override + public String getIssueString(Source source) { + return ctx.getIssueString(source); + } + + @Override + public boolean hasError() { + return ctx.hasError(); + } + + @Override + public Map getIdToOffsetMap() { + return ctx.getIdToOffsetMap(); + } + + @Override + public ValueString newValueString(Node node) { + return ctx.newValueString(node); } } diff --git a/policy/src/main/java/dev/cel/policy/ParserContext.java b/policy/src/main/java/dev/cel/policy/ParserContext.java index 3b68f82a5..385a022c3 100644 --- a/policy/src/main/java/dev/cel/policy/ParserContext.java +++ b/policy/src/main/java/dev/cel/policy/ParserContext.java @@ -15,15 +15,27 @@ package dev.cel.policy; import dev.cel.common.Source; +import dev.cel.policy.CelPolicy.Match; +import dev.cel.policy.CelPolicy.Rule; +import dev.cel.policy.CelPolicy.Variable; import java.util.Map; /** - * ParserContext declares a set of interfaces for creating and managing metadata for parsed - * policies. + * ParserContext declares a set of interfaces for managing metadata, such as node IDs, parsing + * errors and source offsets. */ public interface ParserContext { + + /** + * NextID returns a monotonically increasing identifier for a source fragment. This ID is + * implicitly created and tracked within the CollectMetadata method. + */ long nextId(); + /** + * CollectMetadata records the source position information of a given node, and returns the id + * associated with the source metadata which is returned in the Policy SourceInfo object. + */ long collectMetadata(T node); void reportError(long id, String message); @@ -33,4 +45,21 @@ public interface ParserContext { boolean hasError(); Map getIdToOffsetMap(); + + /** NewString creates a new ValueString from the YAML node. */ + ValueString newValueString(T node); + + /** + * PolicyParserContext declares a set of interfaces for creating and managing metadata + * specifically for {@link CelPolicy}. + */ + interface PolicyParserContext extends ParserContext { + CelPolicy parsePolicy(PolicyParserContext ctx, T node); + + Rule parseRule(PolicyParserContext ctx, CelPolicy.Builder policyBuilder, T node); + + Match parseMatch(PolicyParserContext ctx, CelPolicy.Builder policyBuilder, T node); + + Variable parseVariable(PolicyParserContext ctx, CelPolicy.Builder policyBuilder, T node); + } } diff --git a/policy/src/main/java/dev/cel/policy/ValueString.java b/policy/src/main/java/dev/cel/policy/ValueString.java index b1de15be5..24c56f4ee 100644 --- a/policy/src/main/java/dev/cel/policy/ValueString.java +++ b/policy/src/main/java/dev/cel/policy/ValueString.java @@ -21,9 +21,9 @@ public abstract class ValueString { /** A unique identifier. This is populated by the parser. */ - abstract long id(); + public abstract long id(); - abstract String value(); + public abstract String value(); @AutoValue.Builder abstract static class Builder { @@ -35,13 +35,15 @@ abstract static class Builder { abstract ValueString build(); } + public abstract Builder toBuilder(); + /** Builder for {@link ValueString}. */ - static Builder newBuilder() { + public static Builder newBuilder() { return new AutoValue_ValueString.Builder().setId(0).setValue(""); } /** Creates a new {@link ValueString} instance with the specified ID and string value. */ - static ValueString of(long id, String value) { + public static ValueString of(long id, String value) { return newBuilder().setId(id).setValue(value).build(); } } diff --git a/policy/src/main/java/dev/cel/policy/YamlHelper.java b/policy/src/main/java/dev/cel/policy/YamlHelper.java index a29e8c73b..d340ce329 100644 --- a/policy/src/main/java/dev/cel/policy/YamlHelper.java +++ b/policy/src/main/java/dev/cel/policy/YamlHelper.java @@ -104,19 +104,7 @@ static boolean newBoolean(ParserContext ctx, Node node) { } static String newString(ParserContext ctx, Node node) { - return YamlHelper.newValueString(ctx, node).value(); - } - - static ValueString newValueString(ParserContext ctx, Node node) { - long id = ctx.collectMetadata(node); - if (!assertYamlType(ctx, id, node, YamlNodeType.STRING, YamlNodeType.TEXT)) { - return ValueString.of(id, ERROR); - } - - ScalarNode scalarNode = (ScalarNode) node; - - // TODO: Compute relative source for multiline strings - return ValueString.of(id, scalarNode.getValue()); + return ctx.newValueString(node).value(); } private YamlHelper() {} diff --git a/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java index 31438fd98..613aca537 100644 --- a/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java +++ b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java @@ -15,12 +15,15 @@ package dev.cel.policy; import static com.google.common.collect.ImmutableList.toImmutableList; +import static dev.cel.policy.YamlHelper.ERROR; +import static dev.cel.policy.YamlHelper.assertYamlType; import com.google.common.base.Joiner; import dev.cel.common.CelIssue; import dev.cel.common.CelSourceLocation; import dev.cel.common.Source; import dev.cel.common.internal.CelCodePointArray; +import dev.cel.policy.YamlHelper.YamlNodeType; import java.util.ArrayList; import java.util.HashMap; import java.util.Map; @@ -60,6 +63,19 @@ public Map getIdToOffsetMap() { return idToOffsetMap; } + @Override + public ValueString newValueString(Node node) { + long id = collectMetadata(node); + if (!assertYamlType(this, id, node, YamlNodeType.STRING, YamlNodeType.TEXT)) { + return ValueString.of(id, ERROR); + } + + ScalarNode scalarNode = (ScalarNode) node; + + // TODO: Compute relative source for multiline strings + return ValueString.of(id, scalarNode.getValue()); + } + @Override public long collectMetadata(Node node) { long id = nextId(); diff --git a/policy/src/test/java/dev/cel/policy/BUILD.bazel b/policy/src/test/java/dev/cel/policy/BUILD.bazel index 05154b7e6..a77a76e6b 100644 --- a/policy/src/test/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/test/java/dev/cel/policy/BUILD.bazel @@ -25,9 +25,11 @@ java_library( "//policy:config", "//policy:config_parser", "//policy:parser", + "//policy:parser_context", "//policy:parser_factory", "//policy:source", "//policy:validation_exception", + "//policy:value_string", "//runtime", "@maven//:com_google_api_grpc_proto_google_common_protos", "@maven//:com_google_guava_guava", diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java index 41cefcae1..562044ca3 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java @@ -32,6 +32,7 @@ import dev.cel.extensions.CelOptionalLibrary; import dev.cel.parser.CelStandardMacro; import dev.cel.parser.CelUnparserFactory; +import dev.cel.policy.PolicyTestHelper.K8sTagHandler; import dev.cel.policy.PolicyTestHelper.PolicyTestSuite; import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection; import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection.PolicyTestCase; @@ -46,7 +47,7 @@ public final class CelPolicyCompilerImplTest { private static final CelPolicyParser POLICY_PARSER = - CelPolicyParserFactory.newYamlParserBuilder().build(); + CelPolicyParserFactory.newYamlParserBuilder().addTagVisitor(new K8sTagHandler()).build(); private static final CelPolicyConfigParser POLICY_CONFIG_PARSER = CelPolicyParserFactory.newYamlConfigParser(); private static final CelOptions CEL_OPTIONS = diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java index 6a7cfc800..7e4d8c412 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java @@ -19,6 +19,7 @@ import com.google.testing.junit.testparameterinjector.TestParameter; import com.google.testing.junit.testparameterinjector.TestParameterInjector; +import dev.cel.policy.PolicyTestHelper.K8sTagHandler; import dev.cel.policy.PolicyTestHelper.TestYamlPolicy; import org.junit.Test; import org.junit.runner.RunWith; @@ -27,7 +28,7 @@ public final class CelPolicyYamlParserTest { private static final CelPolicyParser POLICY_PARSER = - CelPolicyParserFactory.newYamlParserBuilder().build(); + CelPolicyParserFactory.newYamlParserBuilder().addTagVisitor(new K8sTagHandler()).build(); @Test public void parseYamlPolicy_success(@TestParameter TestYamlPolicy yamlPolicy) throws Exception { diff --git a/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java b/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java index ed1b94a66..8b7eb1b40 100644 --- a/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java +++ b/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java @@ -19,6 +19,11 @@ import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Ascii; import com.google.common.io.Resources; +import dev.cel.policy.CelPolicy.Match; +import dev.cel.policy.CelPolicy.Match.Result; +import dev.cel.policy.CelPolicy.Rule; +import dev.cel.policy.CelPolicyParser.TagVisitor; +import dev.cel.policy.ParserContext.PolicyParserContext; import java.io.IOException; import java.net.URL; import java.util.List; @@ -26,6 +31,8 @@ import org.yaml.snakeyaml.LoaderOptions; import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.Constructor; +import org.yaml.snakeyaml.nodes.Node; +import org.yaml.snakeyaml.nodes.SequenceNode; /** Package-private class to assist with policy testing. */ final class PolicyTestHelper { @@ -66,8 +73,16 @@ enum TestYamlPolicy { + " variables.matches_dest_ip || variables.matches_dest_label," + " (variables.matches_nationality && variables.matches_dest) ? true :" + " ((!variables.has_nationality && variables.matches_origin_ip &&" - + " variables.matches_dest) ? true : false)))))))"); - + + " variables.matches_dest) ? true : false)))))))"), + K8S( + "k8s", + true, + "cel.bind(variables.env, resource.labels.?environment.orValue(\"prod\")," + + " cel.bind(variables.break_glass, resource.labels.?break_glass.orValue(\"false\") ==" + + " \"true\", !(variables.break_glass || resource.containers.all(c," + + " c.startsWith(variables.env + \".\"))) ? optional.of(\"only \" + variables.env + \"" + + " containers are allowed in namespace \" + resource.namespace) :" + + " optional.none()))"); private final String name; private final boolean producesOptionalResult; private final String unparsed; @@ -221,5 +236,106 @@ private static String readFile(String path) throws IOException { return Resources.toString(getResource(path), UTF_8); } + static class K8sTagHandler implements TagVisitor { + + @Override + public void visitPolicyTag( + PolicyParserContext ctx, + long id, + String tagName, + Node node, + CelPolicy.Builder policyBuilder) { + switch (tagName) { + case "kind": + policyBuilder.putMetadata("kind", ctx.newValueString(node)); + break; + case "metadata": + long metadataId = ctx.collectMetadata(node); + if (!node.getTag().getValue().equals("tag:yaml.org,2002:map")) { + ctx.reportError( + metadataId, + String.format( + "invalid 'metadata' type, expected map got: %s", node.getTag().getValue())); + } + break; + case "spec": + Rule rule = ctx.parseRule(ctx, policyBuilder, node); + policyBuilder.setRule(rule); + break; + default: + TagVisitor.super.visitPolicyTag(ctx, id, tagName, node, policyBuilder); + break; + } + } + + @Override + public void visitRuleTag( + PolicyParserContext ctx, + long id, + String tagName, + Node node, + CelPolicy.Builder policyBuilder, + Rule.Builder ruleBuilder) { + switch (tagName) { + case "failurePolicy": + policyBuilder.putMetadata(tagName, ctx.newValueString(node)); + break; + case "matchConstraints": + long matchConstraintsId = ctx.collectMetadata(node); + if (!node.getTag().getValue().equals("tag:yaml.org,2002:map")) { + ctx.reportError( + matchConstraintsId, + String.format( + "invalid 'matchConstraints' type, expected map got: %s", + node.getTag().getValue())); + } + break; + case "validations": + long validationId = ctx.collectMetadata(node); + if (!node.getTag().getValue().equals("tag:yaml.org,2002:seq")) { + ctx.reportError( + validationId, + String.format( + "invalid 'validations' type, expected list got: %s", node.getTag().getValue())); + } + + SequenceNode validationNodes = (SequenceNode) node; + for (Node element : validationNodes.getValue()) { + ruleBuilder.addMatches(ctx.parseMatch(ctx, policyBuilder, element)); + } + break; + default: + TagVisitor.super.visitRuleTag(ctx, id, tagName, node, policyBuilder, ruleBuilder); + break; + } + } + + @Override + public void visitMatchTag( + PolicyParserContext ctx, + long id, + String tagName, + Node node, + CelPolicy.Builder policyBuilder, + Match.Builder matchBuilder) { + switch (tagName) { + case "expression": + // The K8s expression to validate must return false in order to generate a violation + // message. + ValueString conditionValue = ctx.newValueString(node); + conditionValue = + conditionValue.toBuilder().setValue("!(" + conditionValue.value() + ")").build(); + matchBuilder.setCondition(conditionValue); + break; + case "messageExpression": + matchBuilder.setResult(Result.ofOutput(ctx.newValueString(node))); + break; + default: + TagVisitor.super.visitMatchTag(ctx, id, tagName, node, policyBuilder, matchBuilder); + break; + } + } + } + private PolicyTestHelper() {} } diff --git a/policy/src/test/resources/k8s/config.yaml b/policy/src/test/resources/k8s/config.yaml new file mode 100644 index 000000000..4df8439ea --- /dev/null +++ b/policy/src/test/resources/k8s/config.yaml @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: k8s +extensions: +- name: "strings" + version: 2 +variables: +- name: "resource.labels" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "string" +- name: "resource.containers" + type: + type_name: "list" + params: + - type_name: "string" +- name: "resource.namespace" + type: + type_name: "string" diff --git a/policy/src/test/resources/k8s/policy.yaml b/policy/src/test/resources/k8s/policy.yaml new file mode 100644 index 000000000..9cc9782fa --- /dev/null +++ b/policy/src/test/resources/k8s/policy.yaml @@ -0,0 +1,36 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: k8s +kind: ValidatingAdmissionPolicy +metadata: + name: "policy.cel.dev" +spec: + failurePolicy: Fail + matchConstraints: + resourceRules: + - apiGroups: ["services"] + apiVersions: ["v3"] + operations: ["CREATE", "UPDATE"] + variables: + - name: env + expression: "resource.labels.?environment.orValue('prod')" + - name: break_glass + expression: "resource.labels.?break_glass.orValue('false') == 'true'" + validations: + - expression: > + variables.break_glass || + resource.containers.all(c, c.startsWith(variables.env + '.')) + messageExpression: > + 'only ' + variables.env + ' containers are allowed in namespace ' + resource.namespace diff --git a/policy/src/test/resources/k8s/tests.yaml b/policy/src/test/resources/k8s/tests.yaml new file mode 100644 index 000000000..8585c5efb --- /dev/null +++ b/policy/src/test/resources/k8s/tests.yaml @@ -0,0 +1,31 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description: K8s admission control tests +section: +- name: "invalid" + tests: + - name: "restricted_container" + input: + resource.namespace: + value: "dev.cel" + resource.labels: + value: + environment: "staging" + resource.containers: + value: + - staging.dev.cel.container1 + - staging.dev.cel.container2 + - preprod.dev.cel.container3 + output: "'only staging containers are allowed in namespace dev.cel'" From e85ee166dc090d0d33342fb8eb2b74ac39ed2646 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Wed, 3 Jul 2024 14:17:33 -0700 Subject: [PATCH 23/33] Pull getIssueString logic into CelIssue PiperOrigin-RevId: 649196570 --- .../test/java/dev/cel/checker/CelIssueTest.java | 11 ++++------- .../src/main/java/dev/cel/common/CelIssue.java | 11 +++++++++++ .../dev/cel/common/CelValidationException.java | 14 ++++---------- .../dev/cel/common/CelValidationResult.java | 8 ++------ policy/src/main/java/dev/cel/policy/BUILD.bazel | 6 +++--- .../dev/cel/policy/CelPolicyCompilerImpl.java | 8 +------- .../cel/policy/CelPolicyYamlConfigParser.java | 6 ++++-- .../dev/cel/policy/CelPolicyYamlParser.java | 17 +++++++---------- .../main/java/dev/cel/policy/ParserContext.java | 7 +++---- .../dev/cel/policy/YamlParserContextImpl.java | 16 +++------------- 10 files changed, 42 insertions(+), 62 deletions(-) diff --git a/checker/src/test/java/dev/cel/checker/CelIssueTest.java b/checker/src/test/java/dev/cel/checker/CelIssueTest.java index e0d34fd0c..a200a5fd5 100644 --- a/checker/src/test/java/dev/cel/checker/CelIssueTest.java +++ b/checker/src/test/java/dev/cel/checker/CelIssueTest.java @@ -16,9 +16,7 @@ import static com.google.common.truth.Truth.assertThat; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import dev.cel.common.CelIssue; import dev.cel.common.CelSource; import org.junit.Test; @@ -28,8 +26,6 @@ @RunWith(JUnit4.class) public final class CelIssueTest { - private static final Joiner JOINER = Joiner.on('\n'); - @Test public void toDisplayString_narrow() throws Exception { CelSource source = @@ -38,7 +34,7 @@ public void toDisplayString_narrow() throws Exception { ImmutableList.of( CelIssue.formatError(1, 1, "No such field"), CelIssue.formatError(2, 20, "Syntax error, missing paren")); - assertThat(JOINER.join(Iterables.transform(issues, error -> error.toDisplayString(source)))) + assertThat(CelIssue.toDisplayString(issues, source)) .isEqualTo( "ERROR: issues-test:1:2: No such field\n" + " | a.b\n" @@ -53,7 +49,7 @@ public void toDisplayString_wideAndNarrow() throws Exception { CelSource source = CelSource.newBuilder("你好吗\n我b很好\n").setDescription("issues-test").build(); ImmutableList issues = ImmutableList.of(CelIssue.formatError(2, 3, "Unexpected character '好'")); - assertThat(JOINER.join(Iterables.transform(issues, error -> error.toDisplayString(source)))) + assertThat(CelIssue.toDisplayString(issues, source)) .isEqualTo("ERROR: issues-test:2:4: Unexpected character '好'\n" + " | 我b很好\n" + " | ．.．＾"); } @@ -73,7 +69,8 @@ public void toDisplayString_emojis() throws Exception { + " IDENTIFIER}"), CelIssue.formatError(1, 35, "Syntax error: token recognition error at: '😁'"), CelIssue.formatError(1, 36, "Syntax error: missing IDENTIFIER at ''")); - assertThat(JOINER.join(Iterables.transform(issues, error -> error.toDisplayString(source)))) + + assertThat(CelIssue.toDisplayString(issues, source)) .isEqualTo( "ERROR: issues-test:1:33: Syntax error: extraneous input 'in' expecting {'[', '{'," + " '(', '.', '-', '!', 'true', 'false', 'null', NUM_FLOAT, NUM_INT, NUM_UINT," diff --git a/common/src/main/java/dev/cel/common/CelIssue.java b/common/src/main/java/dev/cel/common/CelIssue.java index 1e3cdb58e..2f507d5a7 100644 --- a/common/src/main/java/dev/cel/common/CelIssue.java +++ b/common/src/main/java/dev/cel/common/CelIssue.java @@ -14,10 +14,14 @@ package dev.cel.common; +import static com.google.common.collect.ImmutableList.toImmutableList; + import com.google.auto.value.AutoValue; +import com.google.common.base.Joiner; import com.google.errorprone.annotations.CheckReturnValue; import com.google.errorprone.annotations.Immutable; import dev.cel.common.internal.SafeStringFormatter; +import java.util.Collection; import java.util.Optional; import java.util.PrimitiveIterator; @@ -28,6 +32,7 @@ @Immutable @SuppressWarnings("UnicodeEscape") // Suppressed to distinguish half-width and full-width chars. public abstract class CelIssue { + private static final Joiner JOINER = Joiner.on('\n'); /** Severity of a CelIssue. */ public enum Severity { @@ -73,6 +78,12 @@ public static CelIssue formatError(int line, int column, String message) { private static final char WIDE_DOT = '\uff0e'; private static final char WIDE_HAT = '\uff3e'; + /** Returns a human-readable error with all issues joined in a single string. */ + public static String toDisplayString(Collection issues, Source source) { + return JOINER.join( + issues.stream().map(iss -> iss.toDisplayString(source)).collect(toImmutableList())); + } + /** Returns a string representing this error that is suitable for displaying to humans. */ public String toDisplayString(Source source) { // Based onhttps://github.com/google/cel-go/blob/v0.5.1/common/error.go#L42. diff --git a/common/src/main/java/dev/cel/common/CelValidationException.java b/common/src/main/java/dev/cel/common/CelValidationException.java index ef136f8a7..522706e83 100644 --- a/common/src/main/java/dev/cel/common/CelValidationException.java +++ b/common/src/main/java/dev/cel/common/CelValidationException.java @@ -15,15 +15,12 @@ package dev.cel.common; import com.google.common.annotations.VisibleForTesting; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import java.util.List; /** Base class for all checked exceptions explicitly thrown by the library during parsing. */ public final class CelValidationException extends CelException { - private static final Joiner JOINER = Joiner.on('\n'); // Truncates all errors beyond this limit in the message. private static final int MAX_ERRORS_TO_REPORT = 1000; @@ -46,17 +43,14 @@ public CelValidationException(CelSource source, List errors) { private static String safeJoinErrorMessage(CelSource source, List errors) { if (errors.size() <= MAX_ERRORS_TO_REPORT) { - return JOINER.join(Iterables.transform(errors, error -> error.toDisplayString(source))); + return CelIssue.toDisplayString(errors, source); } List truncatedErrors = errors.subList(0, MAX_ERRORS_TO_REPORT); - StringBuilder sb = new StringBuilder(); - JOINER.appendTo( - sb, Iterables.transform(truncatedErrors, error -> error.toDisplayString(source))); - sb.append( - String.format("%n...and %d more errors (truncated)", errors.size() - MAX_ERRORS_TO_REPORT)); - return sb.toString(); + return CelIssue.toDisplayString(truncatedErrors, source) + + String.format( + "%n...and %d more errors (truncated)", errors.size() - MAX_ERRORS_TO_REPORT); } /** Returns the {@link CelSource} that was being validated. */ diff --git a/common/src/main/java/dev/cel/common/CelValidationResult.java b/common/src/main/java/dev/cel/common/CelValidationResult.java index 3e52bc8e1..61152c493 100644 --- a/common/src/main/java/dev/cel/common/CelValidationResult.java +++ b/common/src/main/java/dev/cel/common/CelValidationResult.java @@ -17,9 +17,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; import static java.util.Comparator.comparing; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; import com.google.errorprone.annotations.CanIgnoreReturnValue; import com.google.errorprone.annotations.Immutable; import com.google.errorprone.annotations.InlineMe; @@ -33,8 +31,6 @@ @Immutable public final class CelValidationResult { - private static final Joiner JOINER = Joiner.on('\n'); - @SuppressWarnings("Immutable") private final @Nullable Throwable failure; @@ -115,7 +111,7 @@ public ImmutableList getAllIssues() { /** Convert all issues to a human-readable string. */ public String getIssueString() { - return JOINER.join(Iterables.transform(issues, iss -> iss.toDisplayString(source))); + return CelIssue.toDisplayString(issues, source); } /** @@ -131,7 +127,7 @@ public String getDebugString() { /** Convert the {@code CelIssue}s with {@code ERROR} severity to an error string. */ public String getErrorString() { - return JOINER.join(Iterables.transform(getErrors(), error -> error.toDisplayString(source))); + return CelIssue.toDisplayString(getErrors(), source); } private static boolean issueIsError(CelIssue iss) { diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index 407ce2f4e..c13bcf0c4 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -109,7 +109,7 @@ java_library( ":source", ":validation_exception", ":value_string", - "//common:source", + "//common:compiler_common", "//common/internal", "@maven//:com_google_guava_guava", "@maven//:org_yaml_snakeyaml", @@ -217,6 +217,7 @@ java_library( ":parser_context", ":source", ":validation_exception", + "//common:compiler_common", "//common/internal", "@maven//:com_google_guava_guava", "@maven//:org_yaml_snakeyaml", @@ -265,7 +266,7 @@ java_library( deps = [ ":policy", ":value_string", - "//common:source", + "//common:compiler_common", ], ) @@ -302,7 +303,6 @@ java_library( ":value_string", "//common", "//common:compiler_common", - "//common:source", "//common/internal", "@maven//:com_google_guava_guava", "@maven//:org_yaml_snakeyaml", diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java index 4fb81f42b..c2f1516db 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java @@ -15,9 +15,7 @@ package dev.cel.policy; import static com.google.common.base.Preconditions.checkNotNull; -import static com.google.common.collect.ImmutableList.toImmutableList; -import com.google.common.base.Joiner; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; import dev.cel.bundle.Cel; @@ -145,7 +143,6 @@ private static CelAbstractSyntaxTree newErrorAst() { } private static final class CompilerContext { - private static final Joiner JOINER = Joiner.on('\n'); private final ArrayList issues; private final ArrayList newVariableDeclarations; private final CelPolicySource celPolicySource; @@ -170,10 +167,7 @@ private boolean hasError() { } private String getIssueString() { - return JOINER.join( - issues.stream() - .map(iss -> iss.toDisplayString(celPolicySource)) - .collect(toImmutableList())); + return CelIssue.toDisplayString(issues, celPolicySource); } private CompilerContext(CelPolicySource celPolicySource) { diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java index 01dfc77b6..d8955fe2b 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; +import dev.cel.common.CelIssue; import dev.cel.common.internal.CelCodePointArray; import dev.cel.policy.CelPolicyConfig.ExtensionConfig; import dev.cel.policy.CelPolicyConfig.FunctionDecl; @@ -339,8 +340,9 @@ private CelPolicyConfig parseYaml(String source, String description) .setPositionsMap(ctx.getIdToOffsetMap()) .build(); - if (ctx.hasError()) { - throw new CelPolicyValidationException(ctx.getIssueString(configSource)); + if (!ctx.getIssues().isEmpty()) { + throw new CelPolicyValidationException( + CelIssue.toDisplayString(ctx.getIssues(), configSource)); } return policyConfig.setConfigSource(configSource).build(); diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java index 5d36185ab..0972451f4 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java @@ -20,13 +20,14 @@ import static dev.cel.policy.YamlHelper.assertYamlType; import com.google.common.collect.ImmutableSet; -import dev.cel.common.Source; +import dev.cel.common.CelIssue; import dev.cel.common.internal.CelCodePointArray; import dev.cel.policy.CelPolicy.Match; import dev.cel.policy.CelPolicy.Match.Result; import dev.cel.policy.CelPolicy.Variable; import dev.cel.policy.ParserContext.PolicyParserContext; import dev.cel.policy.YamlHelper.YamlNodeType; +import java.util.List; import java.util.Map; import org.yaml.snakeyaml.nodes.MappingNode; import org.yaml.snakeyaml.nodes.Node; @@ -76,8 +77,9 @@ private CelPolicy parseYaml() throws CelPolicyValidationException { CelPolicy celPolicy = parsePolicy(this, node); - if (ctx.hasError()) { - throw new CelPolicyValidationException(ctx.getIssueString(celPolicy.policySource())); + if (!ctx.getIssues().isEmpty()) { + throw new CelPolicyValidationException( + CelIssue.toDisplayString(ctx.getIssues(), celPolicy.policySource())); } return celPolicy; @@ -296,13 +298,8 @@ public void reportError(long id, String message) { } @Override - public String getIssueString(Source source) { - return ctx.getIssueString(source); - } - - @Override - public boolean hasError() { - return ctx.hasError(); + public List getIssues() { + return ctx.getIssues(); } @Override diff --git a/policy/src/main/java/dev/cel/policy/ParserContext.java b/policy/src/main/java/dev/cel/policy/ParserContext.java index 385a022c3..13b816414 100644 --- a/policy/src/main/java/dev/cel/policy/ParserContext.java +++ b/policy/src/main/java/dev/cel/policy/ParserContext.java @@ -14,10 +14,11 @@ package dev.cel.policy; -import dev.cel.common.Source; +import dev.cel.common.CelIssue; import dev.cel.policy.CelPolicy.Match; import dev.cel.policy.CelPolicy.Rule; import dev.cel.policy.CelPolicy.Variable; +import java.util.List; import java.util.Map; /** @@ -40,9 +41,7 @@ public interface ParserContext { void reportError(long id, String message); - String getIssueString(Source source); - - boolean hasError(); + List getIssues(); Map getIdToOffsetMap(); diff --git a/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java index 613aca537..d84b9f830 100644 --- a/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java +++ b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java @@ -14,18 +14,16 @@ package dev.cel.policy; -import static com.google.common.collect.ImmutableList.toImmutableList; import static dev.cel.policy.YamlHelper.ERROR; import static dev.cel.policy.YamlHelper.assertYamlType; -import com.google.common.base.Joiner; import dev.cel.common.CelIssue; import dev.cel.common.CelSourceLocation; -import dev.cel.common.Source; import dev.cel.common.internal.CelCodePointArray; import dev.cel.policy.YamlHelper.YamlNodeType; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; import java.util.Map; import org.yaml.snakeyaml.DumperOptions; import org.yaml.snakeyaml.nodes.Node; @@ -34,8 +32,6 @@ /** Package-private class to assist with storing policy parsing context. */ final class YamlParserContextImpl implements ParserContext { - private static final Joiner JOINER = Joiner.on('\n'); - private final ArrayList issues; private final HashMap idToLocationMap; private final HashMap idToOffsetMap; @@ -48,14 +44,8 @@ public void reportError(long id, String message) { } @Override - public String getIssueString(Source source) { - return JOINER.join( - issues.stream().map(iss -> iss.toDisplayString(source)).collect(toImmutableList())); - } - - @Override - public boolean hasError() { - return !issues.isEmpty(); + public List getIssues() { + return issues; } @Override From 893dfeaa28de3bf5d0b1aba537d48d7430d3fc1d Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Wed, 3 Jul 2024 14:54:49 -0700 Subject: [PATCH 24/33] Internal Changes PiperOrigin-RevId: 649207240 --- policy/BUILD.bazel | 15 +++ .../src/main/java/dev/cel/policy/BUILD.bazel | 109 +++++++++++------- 2 files changed, 80 insertions(+), 44 deletions(-) diff --git a/policy/BUILD.bazel b/policy/BUILD.bazel index 222619d9f..8d7b5d909 100644 --- a/policy/BUILD.bazel +++ b/policy/BUILD.bazel @@ -52,3 +52,18 @@ java_library( name = "compiler_factory", exports = ["//policy/src/main/java/dev/cel/policy:compiler_factory"], ) + +java_library( + name = "parser_builder", + exports = ["//policy/src/main/java/dev/cel/policy:parser_builder"], +) + +java_library( + name = "compiler", + exports = ["//policy/src/main/java/dev/cel/policy:compiler"], +) + +java_library( + name = "compiler_builder", + exports = ["//policy/src/main/java/dev/cel/policy:compiler_builder"], +) diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index c13bcf0c4..e334293da 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -10,6 +10,8 @@ java_library( srcs = [ "CelPolicy.java", ], + tags = [ + ], deps = [ ":required_fields_checker", ":source", @@ -86,6 +88,8 @@ java_library( java_library( name = "parser_factory", srcs = ["CelPolicyParserFactory.java"], + tags = [ + ], deps = [ ":config_parser", ":parser_builder", @@ -121,6 +125,8 @@ java_library( srcs = [ "CelPolicyParser.java", ], + tags = [ + ], deps = [ ":parser_context", ":policy", @@ -133,6 +139,8 @@ java_library( srcs = [ "CelPolicyParserBuilder.java", ], + tags = [ + ], deps = [ ":parser", "@maven//:com_google_errorprone_error_prone_annotations", @@ -144,6 +152,8 @@ java_library( srcs = [ "CelPolicyCompiler.java", ], + tags = [ + ], deps = [ ":policy", ":validation_exception", @@ -156,17 +166,63 @@ java_library( srcs = [ "CelPolicyCompilerBuilder.java", ], + tags = [ + ], deps = [ ":compiler", "@maven//:com_google_errorprone_error_prone_annotations", ], ) +java_library( + name = "compiler_factory", + srcs = ["CelPolicyCompilerFactory.java"], + tags = [ + ], + deps = [ + ":compiler_builder", + ":compiler_impl", + "//bundle:cel", + "//checker:checker_builder", + "//compiler", + "//compiler:compiler_builder", + "//parser:parser_builder", + "//runtime", + ], +) + +java_library( + name = "value_string", + srcs = [ + "ValueString.java", + ], + tags = [ + ], + deps = [ + "//:auto_value", + ], +) + +java_library( + name = "parser_context", + srcs = [ + "ParserContext.java", + ], + tags = [ + ], + deps = [ + ":policy", + ":value_string", + "//common:compiler_common", + ], +) + java_library( name = "compiler_impl", srcs = [ "CelPolicyCompilerImpl.java", ], + visibility = ["//visibility:private"], deps = [ ":compiled_rule", ":compiler", @@ -191,17 +247,14 @@ java_library( ) java_library( - name = "compiler_factory", - srcs = ["CelPolicyCompilerFactory.java"], + name = "required_fields_checker", + srcs = [ + "RequiredFieldsChecker.java", + ], + visibility = ["//visibility:private"], deps = [ - ":compiler_builder", - ":compiler_impl", - "//bundle:cel", - "//checker:checker_builder", - "//compiler", - "//compiler:compiler_builder", - "//parser:parser_builder", - "//runtime", + "//:auto_value", + "@maven//:com_google_guava_guava", ], ) @@ -210,6 +263,7 @@ java_library( srcs = [ "CelPolicyYamlConfigParser.java", ], + visibility = ["//visibility:private"], deps = [ ":common_internal", ":config", @@ -224,31 +278,10 @@ java_library( ], ) -java_library( - name = "value_string", - srcs = [ - "ValueString.java", - ], - deps = [ - "//:auto_value", - ], -) - -java_library( - name = "required_fields_checker", - srcs = [ - "RequiredFieldsChecker.java", - ], - visibility = ["//visibility:private"], - deps = [ - "//:auto_value", - "@maven//:com_google_guava_guava", - ], -) - java_library( name = "compiled_rule", srcs = ["CelCompiledRule.java"], + visibility = ["//visibility:private"], deps = [ "//:auto_value", "//bundle:cel", @@ -258,18 +291,6 @@ java_library( ], ) -java_library( - name = "parser_context", - srcs = [ - "ParserContext.java", - ], - deps = [ - ":policy", - ":value_string", - "//common:compiler_common", - ], -) - java_library( name = "rule_composer", srcs = ["RuleComposer.java"], From 138b11dab2fdcb9060f8d486ad03fefb462a1b48 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Wed, 3 Jul 2024 15:12:12 -0700 Subject: [PATCH 25/33] Introduce protobuf message testing to policies PiperOrigin-RevId: 649212262 --- .../src/test/java/dev/cel/policy/BUILD.bazel | 1 + .../cel/policy/CelPolicyCompilerImplTest.java | 22 +++++++++---- .../java/dev/cel/policy/PolicyTestHelper.java | 7 +++- policy/src/test/resources/pb/config.yaml | 23 +++++++++++++ policy/src/test/resources/pb/policy.yaml | 20 +++++++++++ policy/src/test/resources/pb/tests.yaml | 33 +++++++++++++++++++ 6 files changed, 99 insertions(+), 7 deletions(-) create mode 100644 policy/src/test/resources/pb/config.yaml create mode 100644 policy/src/test/resources/pb/policy.yaml create mode 100644 policy/src/test/resources/pb/tests.yaml diff --git a/policy/src/test/java/dev/cel/policy/BUILD.bazel b/policy/src/test/java/dev/cel/policy/BUILD.bazel index a77a76e6b..8606ca254 100644 --- a/policy/src/test/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/test/java/dev/cel/policy/BUILD.bazel @@ -15,6 +15,7 @@ java_library( "//common", "//common:options", "//common/internal", + "//common/resources/testdata/proto3:test_all_types_java_proto", "//compiler", "//extensions:optional_library", "//parser", diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java index 562044ca3..188e9b199 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java @@ -14,7 +14,7 @@ package dev.cel.policy; -import static com.google.common.collect.ImmutableMap.toImmutableMap; +import static com.google.common.base.Strings.isNullOrEmpty; import static com.google.common.truth.Truth.assertThat; import static dev.cel.policy.PolicyTestHelper.readFromYaml; import static org.junit.Assert.assertThrows; @@ -36,9 +36,11 @@ import dev.cel.policy.PolicyTestHelper.PolicyTestSuite; import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection; import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection.PolicyTestCase; +import dev.cel.policy.PolicyTestHelper.PolicyTestSuite.PolicyTestSection.PolicyTestCase.PolicyTestInput; import dev.cel.policy.PolicyTestHelper.TestYamlPolicy; import dev.cel.runtime.CelRuntime.CelFunctionBinding; -import java.util.Map.Entry; +import dev.cel.testing.testdata.proto3.TestAllTypesProto.TestAllTypes; +import java.util.Map; import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; @@ -138,10 +140,17 @@ public void evaluateYamlPolicy_withCanonicalTestData( // Compile then evaluate the policy CelAbstractSyntaxTree compiledPolicyAst = CelPolicyCompilerFactory.newPolicyCompiler(cel).build().compile(policy); - ImmutableMap input = - testData.testCase.getInput().entrySet().stream() - .collect(toImmutableMap(Entry::getKey, e -> e.getValue().getValue())); - Object evalResult = cel.createProgram(compiledPolicyAst).eval(input); + ImmutableMap.Builder inputBuilder = ImmutableMap.builder(); + for (Map.Entry entry : testData.testCase.getInput().entrySet()) { + String exprInput = entry.getValue().getExpr(); + if (isNullOrEmpty(exprInput)) { + inputBuilder.put(entry.getKey(), entry.getValue().getValue()); + } else { + CelAbstractSyntaxTree exprInputAst = cel.compile(exprInput).getAst(); + inputBuilder.put(entry.getKey(), cel.createProgram(exprInputAst).eval()); + } + } + Object evalResult = cel.createProgram(compiledPolicyAst).eval(inputBuilder.buildOrThrow()); // Assert // Note that policies may either produce an optional or a non-optional result, @@ -219,6 +228,7 @@ private static Cel newCel() { .setStandardMacros(CelStandardMacro.STANDARD_MACROS) .addCompilerLibraries(CelOptionalLibrary.INSTANCE) .addRuntimeLibraries(CelOptionalLibrary.INSTANCE) + .addMessageTypes(TestAllTypes.getDescriptor()) .setOptions(CEL_OPTIONS) .addFunctionBindings( CelFunctionBinding.from( diff --git a/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java b/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java index 8b7eb1b40..f323e5ca5 100644 --- a/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java +++ b/policy/src/test/java/dev/cel/policy/PolicyTestHelper.java @@ -82,7 +82,12 @@ enum TestYamlPolicy { + " \"true\", !(variables.break_glass || resource.containers.all(c," + " c.startsWith(variables.env + \".\"))) ? optional.of(\"only \" + variables.env + \"" + " containers are allowed in namespace \" + resource.namespace) :" - + " optional.none()))"); + + " optional.none()))"), + PB( + "pb", + true, + "(spec.single_int32 > 10) ? optional.of(\"invalid spec, got single_int32=\" +" + + " string(spec.single_int32) + \", wanted <= 10\") : optional.none()"); private final String name; private final boolean producesOptionalResult; private final String unparsed; diff --git a/policy/src/test/resources/pb/config.yaml b/policy/src/test/resources/pb/config.yaml new file mode 100644 index 000000000..d412ec012 --- /dev/null +++ b/policy/src/test/resources/pb/config.yaml @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "pb" +container: "dev.cel.testing.testdata.proto3" +extensions: +- name: "strings" + version: 2 +variables: +- name: "spec" + type: + type_name: "dev.cel.testing.testdata.proto3.TestAllTypes" diff --git a/policy/src/test/resources/pb/policy.yaml b/policy/src/test/resources/pb/policy.yaml new file mode 100644 index 000000000..b26c0dd3b --- /dev/null +++ b/policy/src/test/resources/pb/policy.yaml @@ -0,0 +1,20 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "pb" +rule: + match: + - condition: spec.single_int32 > 10 + output: | + "invalid spec, got single_int32=" + string(spec.single_int32) + ", wanted <= 10" diff --git a/policy/src/test/resources/pb/tests.yaml b/policy/src/test/resources/pb/tests.yaml new file mode 100644 index 000000000..82dd6b11b --- /dev/null +++ b/policy/src/test/resources/pb/tests.yaml @@ -0,0 +1,33 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +description: "Protobuf input tests" +section: +- name: "valid" + tests: + - name: "good spec" + input: + spec: + expr: > + TestAllTypes{single_int32: 10} + output: "optional.none()" +- name: "invalid" + tests: + - name: "bad spec" + input: + spec: + expr: > + TestAllTypes{single_int32: 11} + output: > + "invalid spec, got single_int32=11, wanted <= 10" From 48d5ce6fd119ca5a1849a0974c42315983425070 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Wed, 3 Jul 2024 15:24:10 -0700 Subject: [PATCH 26/33] Validate required fields for policy variables PiperOrigin-RevId: 649215393 --- .../main/java/dev/cel/policy/CelPolicy.java | 16 ++++++++++---- .../dev/cel/policy/CelPolicyYamlParser.java | 12 +++++----- .../cel/policy/CelPolicyYamlParserTest.java | 22 +++++++++++++++++-- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/policy/src/main/java/dev/cel/policy/CelPolicy.java b/policy/src/main/java/dev/cel/policy/CelPolicy.java index 0c14b745f..45a2c666c 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicy.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicy.java @@ -217,20 +217,28 @@ public abstract static class Variable { /** Builder for {@link Variable}. */ @AutoValue.Builder - public abstract static class Builder { + public abstract static class Builder implements RequiredFieldsChecker { + + abstract Optional name(); + + abstract Optional expression(); public abstract Builder setName(ValueString name); public abstract Builder setExpression(ValueString expression); + @Override + public ImmutableList requiredFields() { + return ImmutableList.of( + RequiredField.of("name", this::name), RequiredField.of("expression", this::expression)); + } + public abstract Variable build(); } /** Creates a new builder to construct a {@link Variable} instance. */ public static Builder newBuilder() { - return new AutoValue_CelPolicy_Variable.Builder() - .setName(ValueString.newBuilder().build()) - .setExpression(ValueString.newBuilder().build()); + return new AutoValue_CelPolicy_Variable.Builder(); } } } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java index 0972451f4..b1b96834d 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java @@ -38,13 +38,11 @@ final class CelPolicyYamlParser implements CelPolicyParser { // Sentinel values for parsing errors + private static final ValueString ERROR_VALUE = ValueString.newBuilder().setValue(ERROR).build(); private static final Match ERROR_MATCH = - Match.newBuilder() - .setCondition(ValueString.newBuilder().setValue(ERROR).build()) - .setResult(Result.ofOutput(ValueString.newBuilder().setValue(ERROR).build())) - .build(); + Match.newBuilder().setCondition(ERROR_VALUE).setResult(Result.ofOutput(ERROR_VALUE)).build(); private static final Variable ERROR_VARIABLE = - Variable.newBuilder().setName(ValueString.newBuilder().setValue(ERROR).build()).build(); + Variable.newBuilder().setExpression(ERROR_VALUE).setName(ERROR_VALUE).build(); private final TagVisitor tagVisitor; @@ -272,6 +270,10 @@ public CelPolicy.Variable parseVariable( } } + if (!assertRequiredFields(ctx, id, builder.getMissingRequiredFieldNames())) { + return ERROR_VARIABLE; + } + return builder.build(); } diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java index 7e4d8c412..5bc90cfb9 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyYamlParserTest.java @@ -134,10 +134,28 @@ private enum PolicyParseErrorTestCase { "inputs:\n" + " - name: a\n" + " - name: b", "ERROR: :1:1: Unsupported policy tag: inputs\n" + " | inputs:\n" + " | ^"), UNSUPPORTED_VARIABLE_TAG( - "rule:\n" + " variables:\n" + " - name: 'true'\n" + " alt_name: 'bool_true'", - "ERROR: :4:7: Unsupported variable tag: alt_name\n" + "rule:\n" // + + " variables:\n" // + + " - name: 'hello'\n" // + + " expression: 'true'\n" // + + " alt_name: 'bool_true'", + "ERROR: :5:7: Unsupported variable tag: alt_name\n" + " | alt_name: 'bool_true'\n" + " | ......^"), + MISSING_VARIABLE_NAME( + "rule:\n" // + + " variables:\n" // + + " - expression: 'true'", // + "ERROR: :3:7: Missing required attribute(s): name\n" + + " | - expression: 'true'\n" + + " | ......^"), + MISSING_VARIABLE_EXPRESSION( + "rule:\n" // + + " variables:\n" // + + " - name: 'hello'", // + "ERROR: :3:7: Missing required attribute(s): expression\n" + + " | - name: 'hello'\n" + + " | ......^"), UNSUPPORTED_MATCH_TAG( "rule:\n" + " match:\n" From 393da34c75bf2206b43269214a84526e7f727413 Mon Sep 17 00:00:00 2001 From: Tristan Swadell Date: Mon, 8 Jul 2024 11:20:52 -0700 Subject: [PATCH 27/33] Return `Optional.empty()` rather than Precondition fail When computing source location offsets, a precondition check to verify value lines and columns was causing upstream callers to fail rather than returning the empty optional. The change simply reverts to an empty optional on unsupported input ranges. PiperOrigin-RevId: 650315192 --- common/src/main/java/dev/cel/common/CelSource.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/src/main/java/dev/cel/common/CelSource.java b/common/src/main/java/dev/cel/common/CelSource.java index cc9244e30..1c8d4dbe8 100644 --- a/common/src/main/java/dev/cel/common/CelSource.java +++ b/common/src/main/java/dev/cel/common/CelSource.java @@ -125,8 +125,9 @@ public Optional getSnippet(int line) { */ private static Optional getLocationOffsetImpl( List lineOffsets, int line, int column) { - checkArgument(line > 0); - checkArgument(column >= 0); + if (line <= 0 || column < 0) { + return Optional.empty(); + } int offset = CelSourceHelper.findLineOffset(lineOffsets, line); if (offset == -1) { return Optional.empty(); From 041b37eab4e211114d76f879a6955383ea0106eb Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 8 Jul 2024 12:03:00 -0700 Subject: [PATCH 28/33] Internal Changes PiperOrigin-RevId: 650329658 --- policy/BUILD.bazel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/policy/BUILD.bazel b/policy/BUILD.bazel index 8d7b5d909..a63a0752c 100644 --- a/policy/BUILD.bazel +++ b/policy/BUILD.bazel @@ -1,6 +1,6 @@ package( default_applicable_licenses = ["//:license"], - default_visibility = ["//visibility:public"], # TODO: Expose to public + default_visibility = ["//visibility:public"], ) java_library( From 8321498c554bc33585a9bdb6b45e05db3d9d4023 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Thu, 11 Jul 2024 14:43:46 -0700 Subject: [PATCH 29/33] Compute absolute source location correctly for multiline YAML strings PiperOrigin-RevId: 651540935 --- .../src/main/java/dev/cel/policy/BUILD.bazel | 2 +- .../dev/cel/policy/CelPolicyCompilerImpl.java | 26 +++-- .../cel/policy/CelPolicyYamlConfigParser.java | 9 +- .../dev/cel/policy/CelPolicyYamlParser.java | 17 ++-- .../dev/cel/policy/YamlParserContextImpl.java | 65 ++++++++++--- .../cel/policy/CelPolicyCompilerImplTest.java | 94 ++++++++++++++++++- 6 files changed, 177 insertions(+), 36 deletions(-) diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index e334293da..9e42067fb 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -321,10 +321,10 @@ java_library( visibility = ["//visibility:private"], deps = [ ":parser_context", + ":source", ":value_string", "//common", "//common:compiler_common", - "//common/internal", "@maven//:com_google_guava_guava", "@maven//:org_yaml_snakeyaml", ], diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java index c2f1516db..7ec28d546 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java @@ -149,15 +149,29 @@ private static final class CompilerContext { private void addIssue(long id, List issues) { for (CelIssue issue : issues) { - // Compute relative source and add them into the issues set - int position = Optional.ofNullable(celPolicySource.getPositionsMap().get(id)).orElse(-1); - position += issue.getSourceLocation().getColumn(); - CelSourceLocation loc = - celPolicySource.getOffsetLocation(position).orElse(CelSourceLocation.NONE); - this.issues.add(CelIssue.formatError(loc, issue.getMessage())); + CelSourceLocation absoluteLocation = computeAbsoluteLocation(id, issue); + this.issues.add(CelIssue.formatError(absoluteLocation, issue.getMessage())); } } + private CelSourceLocation computeAbsoluteLocation(long id, CelIssue issue) { + int policySourceOffset = + Optional.ofNullable(celPolicySource.getPositionsMap().get(id)).orElse(-1); + CelSourceLocation policySourceLocation = + celPolicySource.getOffsetLocation(policySourceOffset).orElse(null); + if (policySourceLocation == null) { + return CelSourceLocation.NONE; + } + + int absoluteLine = issue.getSourceLocation().getLine() + policySourceLocation.getLine() - 1; + int absoluteColumn = issue.getSourceLocation().getColumn() + policySourceLocation.getColumn(); + int absoluteOffset = celPolicySource.getContent().lineOffsets().get(absoluteLine - 2); + + return celPolicySource + .getOffsetLocation(absoluteOffset + absoluteColumn) + .orElse(CelSourceLocation.NONE); + } + private void addNewVarDecl(CelVarDecl newVarDecl) { newVariableDeclarations.add(newVarDecl); } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java index d8955fe2b..f0525f7ef 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyYamlConfigParser.java @@ -331,14 +331,13 @@ private CelPolicyConfig parseYaml(String source, String description) throw new CelPolicyValidationException("YAML document is malformed: " + e.getMessage(), e); } - CelCodePointArray configCodePointArray = CelCodePointArray.fromString(source); - ParserContext ctx = YamlParserContextImpl.newInstance(configCodePointArray); - CelPolicyConfig.Builder policyConfig = parseConfig(ctx, node); CelPolicySource configSource = - CelPolicySource.newBuilder(configCodePointArray) + CelPolicySource.newBuilder(CelCodePointArray.fromString(source)) .setDescription(description) - .setPositionsMap(ctx.getIdToOffsetMap()) .build(); + ParserContext ctx = YamlParserContextImpl.newInstance(configSource); + CelPolicyConfig.Builder policyConfig = parseConfig(ctx, node); + configSource = configSource.toBuilder().setPositionsMap(ctx.getIdToOffsetMap()).build(); if (!ctx.getIssues().isEmpty()) { throw new CelPolicyValidationException( diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java index b1b96834d..541dfdfb3 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyYamlParser.java @@ -61,14 +61,13 @@ public CelPolicy parse(String policySource, String description) private static class ParserImpl implements PolicyParserContext { private final TagVisitor tagVisitor; - private final CelCodePointArray policySource; - private final String description; + private final CelPolicySource policySource; private final ParserContext ctx; private CelPolicy parseYaml() throws CelPolicyValidationException { Node node; try { - node = YamlHelper.parseYamlSource(policySource.toString()); + node = YamlHelper.parseYamlSource(policySource.getContent().toString()); } catch (RuntimeException e) { throw new CelPolicyValidationException("YAML document is malformed: " + e.getMessage(), e); } @@ -86,11 +85,9 @@ private CelPolicy parseYaml() throws CelPolicyValidationException { @Override public CelPolicy parsePolicy(PolicyParserContext ctx, Node node) { CelPolicy.Builder policyBuilder = CelPolicy.newBuilder(); - CelPolicySource.Builder sourceBuilder = - CelPolicySource.newBuilder(policySource).setDescription(description); long id = ctx.collectMetadata(node); if (!assertYamlType(ctx, id, node, YamlNodeType.MAP)) { - return policyBuilder.setPolicySource(sourceBuilder.build()).build(); + return policyBuilder.setPolicySource(policySource).build(); } MappingNode rootNode = (MappingNode) node; @@ -117,7 +114,7 @@ public CelPolicy parsePolicy(PolicyParserContext ctx, Node node) { } return policyBuilder - .setPolicySource(sourceBuilder.setPositionsMap(ctx.getIdToOffsetMap()).build()) + .setPolicySource(policySource.toBuilder().setPositionsMap(ctx.getIdToOffsetMap()).build()) .build(); } @@ -279,8 +276,10 @@ public CelPolicy.Variable parseVariable( private ParserImpl(TagVisitor tagVisitor, String source, String description) { this.tagVisitor = tagVisitor; - this.policySource = CelCodePointArray.fromString(source); - this.description = description; + this.policySource = + CelPolicySource.newBuilder(CelCodePointArray.fromString(source)) + .setDescription(description) + .build(); this.ctx = YamlParserContextImpl.newInstance(policySource); } diff --git a/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java index d84b9f830..9dbf77f72 100644 --- a/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java +++ b/policy/src/main/java/dev/cel/policy/YamlParserContextImpl.java @@ -17,15 +17,16 @@ import static dev.cel.policy.YamlHelper.ERROR; import static dev.cel.policy.YamlHelper.assertYamlType; +import com.google.common.base.Strings; import dev.cel.common.CelIssue; import dev.cel.common.CelSourceLocation; -import dev.cel.common.internal.CelCodePointArray; import dev.cel.policy.YamlHelper.YamlNodeType; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import org.yaml.snakeyaml.DumperOptions; +import org.yaml.snakeyaml.DumperOptions.ScalarStyle; import org.yaml.snakeyaml.nodes.Node; import org.yaml.snakeyaml.nodes.ScalarNode; @@ -35,7 +36,7 @@ final class YamlParserContextImpl implements ParserContext { private final ArrayList issues; private final HashMap idToLocationMap; private final HashMap idToOffsetMap; - private final CelCodePointArray policyContent; + private final CelPolicySource policySource; private long id; @Override @@ -61,8 +62,32 @@ public ValueString newValueString(Node node) { } ScalarNode scalarNode = (ScalarNode) node; + ScalarStyle style = scalarNode.getScalarStyle(); + if (style.equals(ScalarStyle.FOLDED) || style.equals(ScalarStyle.LITERAL)) { + CelSourceLocation location = idToLocationMap.get(id); + int line = location.getLine(); + int column = location.getColumn(); + + String indent = Strings.padStart("", column, ' '); + String text = policySource.getSnippet(line).orElse(""); + StringBuilder raw = new StringBuilder(); + while (text.startsWith(indent)) { + line++; + raw.append(text); + text = policySource.getSnippet(line).orElse(""); + if (text.isEmpty()) { + break; + } + if (text.startsWith(indent)) { + raw.append("\n"); + } + } + + idToOffsetMap.compute(id, (k, offset) -> offset - column); + + return ValueString.of(id, raw.toString()); + } - // TODO: Compute relative source for multiline strings return ValueString.of(id, scalarNode.getValue()); } @@ -73,16 +98,34 @@ public long collectMetadata(Node node) { int column = node.getStartMark().getColumn(); if (node instanceof ScalarNode) { DumperOptions.ScalarStyle style = ((ScalarNode) node).getScalarStyle(); - if (style.equals(DumperOptions.ScalarStyle.SINGLE_QUOTED) - || style.equals(DumperOptions.ScalarStyle.DOUBLE_QUOTED)) { - column++; + switch (style) { + case SINGLE_QUOTED: + case DOUBLE_QUOTED: + column++; + break; + case LITERAL: + case FOLDED: + // For multi-lines, actual string content begins on next line + line++; + // Columns must be computed from the indentation + column = 0; + String snippet = policySource.getSnippet(line).orElse(""); + for (char c : snippet.toCharArray()) { + if (!Character.isWhitespace(c)) { + break; + } + column++; + } + break; + default: + break; } } idToLocationMap.put(id, CelSourceLocation.of(line, column)); int offset = 0; if (line > 1) { - offset = policyContent.lineOffsets().get(line - 2) + column; + offset = policySource.getContent().lineOffsets().get(line - 2) + column; } idToOffsetMap.put(id, offset); @@ -94,14 +137,14 @@ public long nextId() { return ++id; } - static ParserContext newInstance(CelCodePointArray policyContent) { - return new YamlParserContextImpl(policyContent); + static ParserContext newInstance(CelPolicySource source) { + return new YamlParserContextImpl(source); } - private YamlParserContextImpl(CelCodePointArray policyContent) { + private YamlParserContextImpl(CelPolicySource source) { this.issues = new ArrayList<>(); this.idToLocationMap = new HashMap<>(); this.idToOffsetMap = new HashMap<>(); - this.policyContent = policyContent; + this.policySource = source; } } diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java index 188e9b199..22a18d69c 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java @@ -104,21 +104,35 @@ public void compileYamlPolicy_containsError_throws() throws Exception { + "ERROR: errors/policy.yaml:23:27: mismatched input '2' expecting {'}', ','}\n" + " | expression: \"{1:305 2:569}\"\n" + " | ..........................^\n" - + "ERROR: errors/policy.yaml:31:65: extraneous input ']' expecting ')'\n" + + "ERROR: errors/policy.yaml:31:75: extraneous input ']' expecting ')'\n" + " | \"missing one or more required labels:" + " %s\".format(variables.missing])\n" - + " | ................................................................^\n" - + "ERROR: errors/policy.yaml:34:57: undeclared reference to 'format' (in container" + + " | ..........................................................................^\n" + + "ERROR: errors/policy.yaml:34:67: undeclared reference to 'format' (in container" + " '')\n" + " | \"invalid values provided on one or more labels:" + " %s\".format([variables.invalid])\n" - + " | ........................................................^\n" + + " | ..................................................................^\n" + "ERROR: errors/policy.yaml:35:24: found no matching overload for '_==_' applied" + " to '(bool, string)' (candidates: (%A0, %A0))\n" + " | - condition: false == \"0\"\n" + " | .......................^"); } + @Test + public void compileYamlPolicy_multilineContainsError_throws( + @TestParameter MultilineErrorTest testCase) throws Exception { + String policyContent = testCase.yaml; + CelPolicy policy = POLICY_PARSER.parse(policyContent); + + CelPolicyValidationException e = + assertThrows( + CelPolicyValidationException.class, + () -> CelPolicyCompilerFactory.newPolicyCompiler(newCel()).build().compile(policy)); + + assertThat(e).hasMessageThat().isEqualTo(testCase.expected); + } + @Test @SuppressWarnings("unchecked") public void evaluateYamlPolicy_withCanonicalTestData( @@ -246,4 +260,76 @@ private static Cel newCel() { })) .build(); } + + private enum MultilineErrorTest { + SINGLE_FOLDED( + "name: \"errors\"\n" + + "rule:\n" + + " match:\n" + + " - output: >\n" + + " 'test'.format(variables.missing])", + "ERROR: :5:40: extraneous input ']' expecting ')'\n" + + " | 'test'.format(variables.missing])\n" + + " | .......................................^"), + DOUBLE_FOLDED( + "name: \"errors\"\n" + + "rule:\n" + + " match:\n" + + " - output: >\n" + + " 'test'.format(\n" + + " variables.missing])", + "ERROR: :6:26: extraneous input ']' expecting ')'\n" + + " | variables.missing])\n" + + " | .........................^"), + TRIPLE_FOLDED( + "name: \"errors\"\n" + + "rule:\n" + + " match:\n" + + " - output: >\n" + + " 'test'.\n" + + " format(\n" + + " variables.missing])", + "ERROR: :7:26: extraneous input ']' expecting ')'\n" + + " | variables.missing])\n" + + " | .........................^"), + SINGLE_LITERAL( + "name: \"errors\"\n" + + "rule:\n" + + " match:\n" + + " - output: |\n" + + " 'test'.format(variables.missing])", + "ERROR: :5:40: extraneous input ']' expecting ')'\n" + + " | 'test'.format(variables.missing])\n" + + " | .......................................^"), + DOUBLE_LITERAL( + "name: \"errors\"\n" + + "rule:\n" + + " match:\n" + + " - output: |\n" + + " 'test'.format(\n" + + " variables.missing])", + "ERROR: :6:26: extraneous input ']' expecting ')'\n" + + " | variables.missing])\n" + + " | .........................^"), + TRIPLE_LITERAL( + "name: \"errors\"\n" + + "rule:\n" + + " match:\n" + + " - output: |\n" + + " 'test'.\n" + + " format(\n" + + " variables.missing])", + "ERROR: :7:26: extraneous input ']' expecting ')'\n" + + " | variables.missing])\n" + + " | .........................^"), + ; + + private final String yaml; + private final String expected; + + MultilineErrorTest(String yaml, String expected) { + this.yaml = yaml; + this.expected = expected; + } + } } From e7c246bb758e71d7eaf5fe9b7a5e8ca8d9990bae Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Thu, 11 Jul 2024 15:19:04 -0700 Subject: [PATCH 30/33] Make iteration limit configurable for rule composer PiperOrigin-RevId: 651551461 --- .../cel/policy/CelPolicyCompilerBuilder.java | 8 +++++++ .../dev/cel/policy/CelPolicyCompilerImpl.java | 24 +++++++++++++++---- .../java/dev/cel/policy/RuleComposer.java | 15 ++++++++---- 3 files changed, 38 insertions(+), 9 deletions(-) diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerBuilder.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerBuilder.java index a56054e03..13bac2885 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerBuilder.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerBuilder.java @@ -20,9 +20,17 @@ /** Interface for building an instance of {@link CelPolicyCompiler} */ public interface CelPolicyCompilerBuilder { + /** Sets the prefix for the policy variables. Default is `variables.`. */ @CanIgnoreReturnValue CelPolicyCompilerBuilder setVariablesPrefix(String prefix); + /** + * Limit the number of iteration while composing rules into a single AST. An exception is thrown + * if the iteration count exceeds the set value. + */ + @CanIgnoreReturnValue + CelPolicyCompilerBuilder setIterationLimit(int iterationLimit); + @CheckReturnValue CelPolicyCompiler build(); } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java index 7ec28d546..7c9e8dcce 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java @@ -44,8 +44,10 @@ /** Package-private implementation for policy compiler. */ final class CelPolicyCompilerImpl implements CelPolicyCompiler { private static final String DEFAULT_VARIABLE_PREFIX = "variables."; + private static final int DEFAULT_ITERATION_LIMIT = 1000; private final Cel cel; private final String variablesPrefix; + private final int iterationLimit; @Override public CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidationException { @@ -59,7 +61,10 @@ public CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidatio CelOptimizerFactory.standardCelOptimizerBuilder(compiledRule.cel()) .addAstOptimizers( RuleComposer.newInstance( - compiledRule, compilerContext.newVariableDeclarations, variablesPrefix)) + compiledRule, + compilerContext.newVariableDeclarations, + variablesPrefix, + iterationLimit)) .build(); CelAbstractSyntaxTree ast; @@ -194,6 +199,7 @@ private CompilerContext(CelPolicySource celPolicySource) { static final class Builder implements CelPolicyCompilerBuilder { private final Cel cel; private String variablesPrefix; + private int iterationLimit; private Builder(Cel cel) { this.cel = cel; @@ -206,18 +212,28 @@ public Builder setVariablesPrefix(String prefix) { return this; } + @Override + @CanIgnoreReturnValue + public Builder setIterationLimit(int iterationLimit) { + this.iterationLimit = iterationLimit; + return this; + } + @Override public CelPolicyCompiler build() { - return new CelPolicyCompilerImpl(cel, this.variablesPrefix); + return new CelPolicyCompilerImpl(cel, this.variablesPrefix, this.iterationLimit); } } static Builder newBuilder(Cel cel) { - return new Builder(cel).setVariablesPrefix(DEFAULT_VARIABLE_PREFIX); + return new Builder(cel) + .setVariablesPrefix(DEFAULT_VARIABLE_PREFIX) + .setIterationLimit(DEFAULT_ITERATION_LIMIT); } - private CelPolicyCompilerImpl(Cel cel, String variablesPrefix) { + private CelPolicyCompilerImpl(Cel cel, String variablesPrefix, int iterationLimit) { this.cel = checkNotNull(cel); this.variablesPrefix = checkNotNull(variablesPrefix); + this.iterationLimit = iterationLimit; } } diff --git a/policy/src/main/java/dev/cel/policy/RuleComposer.java b/policy/src/main/java/dev/cel/policy/RuleComposer.java index 9650b04a7..c449bf7ac 100644 --- a/policy/src/main/java/dev/cel/policy/RuleComposer.java +++ b/policy/src/main/java/dev/cel/policy/RuleComposer.java @@ -34,7 +34,6 @@ /** Package-private class for composing various rules into a single expression using optimizer. */ final class RuleComposer implements CelAstOptimizer { - private static final int AST_MUTATOR_ITERATION_LIMIT = 1000; private final CelCompiledRule compiledRule; private final ImmutableList newVarDecls; private final String variablePrefix; @@ -121,15 +120,21 @@ private RuleOptimizationResult optimizeRule(CelCompiledRule compiledRule) { } static RuleComposer newInstance( - CelCompiledRule compiledRule, List newVarDecls, String variablePrefix) { - return new RuleComposer(compiledRule, newVarDecls, variablePrefix); + CelCompiledRule compiledRule, + List newVarDecls, + String variablePrefix, + int iterationLimit) { + return new RuleComposer(compiledRule, newVarDecls, variablePrefix, iterationLimit); } private RuleComposer( - CelCompiledRule compiledRule, List newVarDecls, String variablePrefix) { + CelCompiledRule compiledRule, + List newVarDecls, + String variablePrefix, + int iterationLimit) { this.compiledRule = checkNotNull(compiledRule); this.newVarDecls = ImmutableList.copyOf(checkNotNull(newVarDecls)); this.variablePrefix = variablePrefix; - this.astMutator = AstMutator.newInstance(AST_MUTATOR_ITERATION_LIMIT); + this.astMutator = AstMutator.newInstance(iterationLimit); } } From d20d37787b91d6f3282f35484270b08374aade00 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 15 Jul 2024 10:22:53 -0700 Subject: [PATCH 31/33] Decompose policy compile API into rule compilation and composition PiperOrigin-RevId: 652523792 --- policy/BUILD.bazel | 5 ++ .../src/main/java/dev/cel/policy/BUILD.bazel | 28 +++++----- .../java/dev/cel/policy/CelCompiledRule.java | 52 ++++++++++++------- .../dev/cel/policy/CelPolicyCompiler.java | 23 ++++++-- .../dev/cel/policy/CelPolicyCompilerImpl.java | 30 +++++------ .../java/dev/cel/policy/RuleComposer.java | 25 ++++----- 6 files changed, 96 insertions(+), 67 deletions(-) diff --git a/policy/BUILD.bazel b/policy/BUILD.bazel index a63a0752c..4773984b4 100644 --- a/policy/BUILD.bazel +++ b/policy/BUILD.bazel @@ -8,6 +8,11 @@ java_library( exports = ["//policy/src/main/java/dev/cel/policy"], ) +java_library( + name = "compiled_rule", + exports = ["//policy/src/main/java/dev/cel/policy:compiled_rule"], +) + java_library( name = "value_string", exports = ["//policy/src/main/java/dev/cel/policy:value_string"], diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index 9e42067fb..562f14722 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -155,6 +155,7 @@ java_library( tags = [ ], deps = [ + ":compiled_rule", ":policy", ":validation_exception", "//common", @@ -217,6 +218,19 @@ java_library( ], ) +java_library( + name = "compiled_rule", + srcs = ["CelCompiledRule.java"], + deps = [ + "//:auto_value", + "//bundle:cel", + "//common", + "//common:compiler_common", + "@maven//:com_google_errorprone_error_prone_annotations", + "@maven//:com_google_guava_guava", + ], +) + java_library( name = "compiler_impl", srcs = [ @@ -278,19 +292,6 @@ java_library( ], ) -java_library( - name = "compiled_rule", - srcs = ["CelCompiledRule.java"], - visibility = ["//visibility:private"], - deps = [ - "//:auto_value", - "//bundle:cel", - "//common", - "@maven//:com_google_errorprone_error_prone_annotations", - "@maven//:com_google_guava_guava", - ], -) - java_library( name = "rule_composer", srcs = ["RuleComposer.java"], @@ -300,7 +301,6 @@ java_library( "//:auto_value", "//bundle:cel", "//common", - "//common:compiler_common", "//common:mutable_ast", "//common/ast", "//extensions:optional_library", diff --git a/policy/src/main/java/dev/cel/policy/CelCompiledRule.java b/policy/src/main/java/dev/cel/policy/CelCompiledRule.java index f7ee8340d..a5fd23515 100644 --- a/policy/src/main/java/dev/cel/policy/CelCompiledRule.java +++ b/policy/src/main/java/dev/cel/policy/CelCompiledRule.java @@ -19,40 +19,55 @@ import com.google.common.collect.ImmutableList; import dev.cel.bundle.Cel; import dev.cel.common.CelAbstractSyntaxTree; +import dev.cel.common.CelVarDecl; -/** Abstract representation of a compiled rule. */ +/** + * Abstract representation of a compiled rule. This contains set of compiled variables and match + * statements which defines an expression graph for a policy. + */ @AutoValue -abstract class CelCompiledRule { - abstract ImmutableList variables(); +public abstract class CelCompiledRule { + public abstract ImmutableList variables(); - abstract ImmutableList matches(); + public abstract ImmutableList matches(); - abstract Cel cel(); + public abstract Cel cel(); + /** + * A compiled policy variable (ex: variables.foo). Note that this is not the same thing as the + * variables declared in the config. + */ @AutoValue - abstract static class CelCompiledVariable { - abstract String name(); + public abstract static class CelCompiledVariable { + public abstract String name(); - abstract CelAbstractSyntaxTree ast(); + /** Compiled variable in AST. */ + public abstract CelAbstractSyntaxTree ast(); - static CelCompiledVariable create(String name, CelAbstractSyntaxTree ast) { - return new AutoValue_CelCompiledRule_CelCompiledVariable(name, ast); + /** The variable declaration used to compile this variable in {@link #ast}. */ + public abstract CelVarDecl celVarDecl(); + + static CelCompiledVariable create( + String name, CelAbstractSyntaxTree ast, CelVarDecl celVarDecl) { + return new AutoValue_CelCompiledRule_CelCompiledVariable(name, ast, celVarDecl); } } + /** A compiled Match. */ @AutoValue - abstract static class CelCompiledMatch { - abstract CelAbstractSyntaxTree condition(); + public abstract static class CelCompiledMatch { + public abstract CelAbstractSyntaxTree condition(); - abstract Result result(); + public abstract Result result(); + /** Encapsulates the result of this match when condition is met. (either an output or a rule) */ @AutoOneOf(CelCompiledMatch.Result.Kind.class) - abstract static class Result { - abstract CelAbstractSyntaxTree output(); + public abstract static class Result { + public abstract CelAbstractSyntaxTree output(); - abstract CelCompiledRule rule(); + public abstract CelCompiledRule rule(); - abstract Kind kind(); + public abstract Kind kind(); static Result ofOutput(CelAbstractSyntaxTree value) { return AutoOneOf_CelCompiledRule_CelCompiledMatch_Result.output(value); @@ -62,7 +77,8 @@ static Result ofRule(CelCompiledRule value) { return AutoOneOf_CelCompiledRule_CelCompiledMatch_Result.rule(value); } - enum Kind { + /** Kind for {@link Result}. */ + public enum Kind { OUTPUT, RULE } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java index 6a63f0389..b1dae87e5 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java @@ -20,8 +20,25 @@ public interface CelPolicyCompiler { /** - * Generates a single CEL AST from a collection of policy expressions associated with a CEL - * environment. + * Combines the {@link #compileRule} and {@link #compose} into a single call. + * + *

This generates a single CEL AST from a collection of policy expressions associated with a + * CEL environment. */ - CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidationException; + default CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidationException { + return compose(compileRule(policy)); + } + + /** + * Produces a {@link CelCompiledRule} from the policy which contains a set of compiled variables + * and match statements. Compiled rule defines an expression graph, which can be composed into a + * single expression via {@link #compose} call. + */ + CelCompiledRule compileRule(CelPolicy policy) throws CelPolicyValidationException; + + /** + * Composes {@link CelCompiledRule}, representing an expression graph, into a single expression + * value. + */ + CelAbstractSyntaxTree compose(CelCompiledRule compiledRule) throws CelPolicyValidationException; } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java index 7c9e8dcce..f8884a911 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java @@ -50,21 +50,23 @@ final class CelPolicyCompilerImpl implements CelPolicyCompiler { private final int iterationLimit; @Override - public CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidationException { + public CelCompiledRule compileRule(CelPolicy policy) throws CelPolicyValidationException { CompilerContext compilerContext = new CompilerContext(policy.policySource()); - CelCompiledRule compiledRule = compileRule(policy.rule(), cel, compilerContext); + CelCompiledRule compiledRule = compileRuleImpl(policy.rule(), cel, compilerContext); if (compilerContext.hasError()) { throw new CelPolicyValidationException(compilerContext.getIssueString()); } + return compiledRule; + } + + @Override + public CelAbstractSyntaxTree compose(CelCompiledRule compiledRule) + throws CelPolicyValidationException { CelOptimizer optimizer = CelOptimizerFactory.standardCelOptimizerBuilder(compiledRule.cel()) .addAstOptimizers( - RuleComposer.newInstance( - compiledRule, - compilerContext.newVariableDeclarations, - variablesPrefix, - iterationLimit)) + RuleComposer.newInstance(compiledRule, variablesPrefix, iterationLimit)) .build(); CelAbstractSyntaxTree ast; @@ -82,7 +84,7 @@ public CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidatio return ast; } - private CelCompiledRule compileRule( + private CelCompiledRule compileRuleImpl( CelPolicy.Rule rule, Cel ruleCel, CompilerContext compilerContext) { ImmutableList.Builder variableBuilder = ImmutableList.builder(); for (Variable variable : rule.variables()) { @@ -100,9 +102,8 @@ private CelCompiledRule compileRule( String variableName = variable.name().value(); CelVarDecl newVariable = CelVarDecl.newVarDeclaration(variablesPrefix + variableName, outputType); - compilerContext.addNewVarDecl(newVariable); ruleCel = ruleCel.toCelBuilder().addVarDeclarations(newVariable).build(); - variableBuilder.add(CelCompiledVariable.create(variableName, varAst)); + variableBuilder.add(CelCompiledVariable.create(variableName, varAst, newVariable)); } ImmutableList.Builder matchBuilder = ImmutableList.builder(); @@ -129,7 +130,8 @@ private CelCompiledRule compileRule( matchResult = Result.ofOutput(outputAst); break; case RULE: - CelCompiledRule nestedRule = compileRule(match.result().rule(), ruleCel, compilerContext); + CelCompiledRule nestedRule = + compileRuleImpl(match.result().rule(), ruleCel, compilerContext); matchResult = Result.ofRule(nestedRule); break; default: @@ -149,7 +151,6 @@ private static CelAbstractSyntaxTree newErrorAst() { private static final class CompilerContext { private final ArrayList issues; - private final ArrayList newVariableDeclarations; private final CelPolicySource celPolicySource; private void addIssue(long id, List issues) { @@ -177,10 +178,6 @@ private CelSourceLocation computeAbsoluteLocation(long id, CelIssue issue) { .orElse(CelSourceLocation.NONE); } - private void addNewVarDecl(CelVarDecl newVarDecl) { - newVariableDeclarations.add(newVarDecl); - } - private boolean hasError() { return !issues.isEmpty(); } @@ -191,7 +188,6 @@ private String getIssueString() { private CompilerContext(CelPolicySource celPolicySource) { this.issues = new ArrayList<>(); - this.newVariableDeclarations = new ArrayList<>(); this.celPolicySource = celPolicySource; } } diff --git a/policy/src/main/java/dev/cel/policy/RuleComposer.java b/policy/src/main/java/dev/cel/policy/RuleComposer.java index c449bf7ac..77a82b73c 100644 --- a/policy/src/main/java/dev/cel/policy/RuleComposer.java +++ b/policy/src/main/java/dev/cel/policy/RuleComposer.java @@ -15,6 +15,7 @@ package dev.cel.policy; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.auto.value.AutoValue; import com.google.common.collect.ImmutableList; @@ -22,7 +23,6 @@ import dev.cel.bundle.Cel; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelMutableAst; -import dev.cel.common.CelVarDecl; import dev.cel.common.ast.CelConstant.Kind; import dev.cel.extensions.CelOptionalLibrary.Function; import dev.cel.optimizer.AstMutator; @@ -30,19 +30,22 @@ import dev.cel.parser.Operator; import dev.cel.policy.CelCompiledRule.CelCompiledMatch; import dev.cel.policy.CelCompiledRule.CelCompiledVariable; -import java.util.List; /** Package-private class for composing various rules into a single expression using optimizer. */ final class RuleComposer implements CelAstOptimizer { private final CelCompiledRule compiledRule; - private final ImmutableList newVarDecls; private final String variablePrefix; private final AstMutator astMutator; @Override public OptimizationResult optimize(CelAbstractSyntaxTree ast, Cel cel) { RuleOptimizationResult result = optimizeRule(compiledRule); - return OptimizationResult.create(result.ast().toParsedAst(), newVarDecls, ImmutableList.of()); + return OptimizationResult.create( + result.ast().toParsedAst(), + compiledRule.variables().stream() + .map(CelCompiledVariable::celVarDecl) + .collect(toImmutableList()), + ImmutableList.of()); } @AutoValue @@ -120,20 +123,12 @@ private RuleOptimizationResult optimizeRule(CelCompiledRule compiledRule) { } static RuleComposer newInstance( - CelCompiledRule compiledRule, - List newVarDecls, - String variablePrefix, - int iterationLimit) { - return new RuleComposer(compiledRule, newVarDecls, variablePrefix, iterationLimit); + CelCompiledRule compiledRule, String variablePrefix, int iterationLimit) { + return new RuleComposer(compiledRule, variablePrefix, iterationLimit); } - private RuleComposer( - CelCompiledRule compiledRule, - List newVarDecls, - String variablePrefix, - int iterationLimit) { + private RuleComposer(CelCompiledRule compiledRule, String variablePrefix, int iterationLimit) { this.compiledRule = checkNotNull(compiledRule); - this.newVarDecls = ImmutableList.copyOf(checkNotNull(newVarDecls)); this.variablePrefix = variablePrefix; this.astMutator = AstMutator.newInstance(iterationLimit); } From 9f78ec3fc0ce4a52f8593bb06af5c62d240187e9 Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 15 Jul 2024 15:51:38 -0700 Subject: [PATCH 32/33] Surface type-mismatch errors in a readable fashion during rule composition PiperOrigin-RevId: 652621800 --- .../src/main/java/dev/cel/policy/BUILD.bazel | 3 + .../java/dev/cel/policy/CelCompiledRule.java | 28 ++++++- .../dev/cel/policy/CelPolicyCompiler.java | 5 +- .../dev/cel/policy/CelPolicyCompilerImpl.java | 50 +++++++++++-- .../java/dev/cel/policy/RuleComposer.java | 75 ++++++++++++++++--- .../cel/policy/CelPolicyCompilerImplTest.java | 72 +++++++++--------- policy/src/test/resources/BUILD.bazel | 2 +- .../{errors => compile_errors}/config.yaml | 0 .../compile_errors/expected_errors.baseline | 24 ++++++ .../{errors => compile_errors}/policy.yaml | 5 +- .../config.yaml | 22 ++++++ .../expected_errors.baseline | 6 ++ .../policy.yaml | 23 ++++++ .../config.yaml | 22 ++++++ .../expected_errors.baseline | 3 + .../policy.yaml | 37 +++++++++ 16 files changed, 316 insertions(+), 61 deletions(-) rename policy/src/test/resources/{errors => compile_errors}/config.yaml (100%) create mode 100644 policy/src/test/resources/compile_errors/expected_errors.baseline rename policy/src/test/resources/{errors => compile_errors}/policy.yaml (92%) create mode 100644 policy/src/test/resources/compose_errors_conflicting_output/config.yaml create mode 100644 policy/src/test/resources/compose_errors_conflicting_output/expected_errors.baseline create mode 100644 policy/src/test/resources/compose_errors_conflicting_output/policy.yaml create mode 100644 policy/src/test/resources/compose_errors_conflicting_subrule/config.yaml create mode 100644 policy/src/test/resources/compose_errors_conflicting_subrule/expected_errors.baseline create mode 100644 policy/src/test/resources/compose_errors_conflicting_subrule/policy.yaml diff --git a/policy/src/main/java/dev/cel/policy/BUILD.bazel b/policy/src/main/java/dev/cel/policy/BUILD.bazel index 562f14722..15e2977c2 100644 --- a/policy/src/main/java/dev/cel/policy/BUILD.bazel +++ b/policy/src/main/java/dev/cel/policy/BUILD.bazel @@ -222,6 +222,7 @@ java_library( name = "compiled_rule", srcs = ["CelCompiledRule.java"], deps = [ + ":value_string", "//:auto_value", "//bundle:cel", "//common", @@ -301,12 +302,14 @@ java_library( "//:auto_value", "//bundle:cel", "//common", + "//common:compiler_common", "//common:mutable_ast", "//common/ast", "//extensions:optional_library", "//optimizer:ast_optimizer", "//optimizer:mutable_ast", "//parser:operator", + "//policy:value_string", "@maven//:com_google_errorprone_error_prone_annotations", "@maven//:com_google_guava_guava", ], diff --git a/policy/src/main/java/dev/cel/policy/CelCompiledRule.java b/policy/src/main/java/dev/cel/policy/CelCompiledRule.java index a5fd23515..4c39fee94 100644 --- a/policy/src/main/java/dev/cel/policy/CelCompiledRule.java +++ b/policy/src/main/java/dev/cel/policy/CelCompiledRule.java @@ -20,6 +20,7 @@ import dev.cel.bundle.Cel; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelVarDecl; +import java.util.Optional; /** * Abstract representation of a compiled rule. This contains set of compiled variables and match @@ -27,6 +28,8 @@ */ @AutoValue public abstract class CelCompiledRule { + public abstract Optional id(); + public abstract ImmutableList variables(); public abstract ImmutableList matches(); @@ -63,14 +66,15 @@ public abstract static class CelCompiledMatch { /** Encapsulates the result of this match when condition is met. (either an output or a rule) */ @AutoOneOf(CelCompiledMatch.Result.Kind.class) public abstract static class Result { - public abstract CelAbstractSyntaxTree output(); + public abstract OutputValue output(); public abstract CelCompiledRule rule(); public abstract Kind kind(); - static Result ofOutput(CelAbstractSyntaxTree value) { - return AutoOneOf_CelCompiledRule_CelCompiledMatch_Result.output(value); + static Result ofOutput(long id, CelAbstractSyntaxTree ast) { + return AutoOneOf_CelCompiledRule_CelCompiledMatch_Result.output( + OutputValue.create(id, ast)); } static Result ofRule(CelCompiledRule value) { @@ -84,6 +88,21 @@ public enum Kind { } } + /** + * Encapsulates the output value of the match with its original ID that was used to compile + * with. + */ + @AutoValue + public abstract static class OutputValue { + public abstract long id(); + + public abstract CelAbstractSyntaxTree ast(); + + public static OutputValue create(long id, CelAbstractSyntaxTree ast) { + return new AutoValue_CelCompiledRule_CelCompiledMatch_OutputValue(id, ast); + } + } + static CelCompiledMatch create( CelAbstractSyntaxTree condition, CelCompiledMatch.Result result) { return new AutoValue_CelCompiledRule_CelCompiledMatch(condition, result); @@ -91,9 +110,10 @@ static CelCompiledMatch create( } static CelCompiledRule create( + Optional id, ImmutableList variables, ImmutableList matches, Cel cel) { - return new AutoValue_CelCompiledRule(variables, matches, cel); + return new AutoValue_CelCompiledRule(id, variables, matches, cel); } } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java index b1dae87e5..e0af6d85b 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompiler.java @@ -26,7 +26,7 @@ public interface CelPolicyCompiler { * CEL environment. */ default CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidationException { - return compose(compileRule(policy)); + return compose(policy, compileRule(policy)); } /** @@ -40,5 +40,6 @@ default CelAbstractSyntaxTree compile(CelPolicy policy) throws CelPolicyValidati * Composes {@link CelCompiledRule}, representing an expression graph, into a single expression * value. */ - CelAbstractSyntaxTree compose(CelCompiledRule compiledRule) throws CelPolicyValidationException; + CelAbstractSyntaxTree compose(CelPolicy policy, CelCompiledRule compiledRule) + throws CelPolicyValidationException; } diff --git a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java index f8884a911..8ca38dad4 100644 --- a/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java +++ b/policy/src/main/java/dev/cel/policy/CelPolicyCompilerImpl.java @@ -15,6 +15,7 @@ package dev.cel.policy; import static com.google.common.base.Preconditions.checkNotNull; +import static com.google.common.collect.ImmutableList.toImmutableList; import com.google.common.collect.ImmutableList; import com.google.errorprone.annotations.CanIgnoreReturnValue; @@ -37,7 +38,9 @@ import dev.cel.policy.CelCompiledRule.CelCompiledVariable; import dev.cel.policy.CelPolicy.Match; import dev.cel.policy.CelPolicy.Variable; +import dev.cel.policy.RuleComposer.RuleCompositionException; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -61,7 +64,7 @@ public CelCompiledRule compileRule(CelPolicy policy) throws CelPolicyValidationE } @Override - public CelAbstractSyntaxTree compose(CelCompiledRule compiledRule) + public CelAbstractSyntaxTree compose(CelPolicy policy, CelCompiledRule compiledRule) throws CelPolicyValidationException { CelOptimizer optimizer = CelOptimizerFactory.standardCelOptimizerBuilder(compiledRule.cel()) @@ -77,8 +80,28 @@ public CelAbstractSyntaxTree compose(CelCompiledRule compiledRule) ast = cel.compile("true").getAst(); ast = optimizer.optimize(ast); } catch (CelValidationException | CelOptimizationException e) { - // TODO: Surface these errors better - throw new CelPolicyValidationException("Failed composing the rules", e); + if (e.getCause() instanceof RuleCompositionException) { + RuleCompositionException re = (RuleCompositionException) e.getCause(); + CompilerContext compilerContext = new CompilerContext(policy.policySource()); + // The exact CEL error message produced from composition failure isn't too useful for users. + // Ex: ERROR: :1:1: found no matching overload for '_?_:_' applied to '(bool, map(int, int), + // bool)' (candidates: (bool, %A0, %A0)) + // Transform the error messages in a user-friendly way while retaining the original + // CelValidationException as its originating cause. + + ImmutableList transformedIssues = + re.compileException.getErrors().stream() + .map(x -> CelIssue.formatError(x.getSourceLocation(), re.failureReason)) + .collect(toImmutableList()); + for (long id : re.errorIds) { + compilerContext.addIssue(id, transformedIssues); + } + + throw new CelPolicyValidationException(compilerContext.getIssueString(), re.getCause()); + } + + // Something has gone seriously wrong. + throw new CelPolicyValidationException("Unexpected error while composing rules.", e); } return ast; @@ -111,6 +134,11 @@ private CelCompiledRule compileRuleImpl( CelAbstractSyntaxTree conditionAst; try { conditionAst = ruleCel.compile(match.condition().value()).getAst(); + if (!conditionAst.getResultType().equals(SimpleType.BOOL)) { + compilerContext.addIssue( + match.condition().id(), + CelIssue.formatError(1, 0, "condition must produce a boolean output.")); + } } catch (CelValidationException e) { compilerContext.addIssue(match.condition().id(), e.getErrors()); continue; @@ -120,14 +148,15 @@ private CelCompiledRule compileRuleImpl( switch (match.result().kind()) { case OUTPUT: CelAbstractSyntaxTree outputAst; + ValueString output = match.result().output(); try { - outputAst = ruleCel.compile(match.result().output().value()).getAst(); + outputAst = ruleCel.compile(output.value()).getAst(); } catch (CelValidationException e) { - compilerContext.addIssue(match.result().output().id(), e.getErrors()); + compilerContext.addIssue(output.id(), e.getErrors()); continue; } - matchResult = Result.ofOutput(outputAst); + matchResult = Result.ofOutput(output.id(), outputAst); break; case RULE: CelCompiledRule nestedRule = @@ -141,7 +170,7 @@ private CelCompiledRule compileRuleImpl( matchBuilder.add(CelCompiledMatch.create(conditionAst, matchResult)); } - return CelCompiledRule.create(variableBuilder.build(), matchBuilder.build(), cel); + return CelCompiledRule.create(rule.id(), variableBuilder.build(), matchBuilder.build(), cel); } private static CelAbstractSyntaxTree newErrorAst() { @@ -153,6 +182,10 @@ private static final class CompilerContext { private final ArrayList issues; private final CelPolicySource celPolicySource; + private void addIssue(long id, CelIssue... issues) { + addIssue(id, Arrays.asList(issues)); + } + private void addIssue(long id, List issues) { for (CelIssue issue : issues) { CelSourceLocation absoluteLocation = computeAbsoluteLocation(id, issue); @@ -163,6 +196,9 @@ private void addIssue(long id, List issues) { private CelSourceLocation computeAbsoluteLocation(long id, CelIssue issue) { int policySourceOffset = Optional.ofNullable(celPolicySource.getPositionsMap().get(id)).orElse(-1); + if (policySourceOffset == -1) { + return CelSourceLocation.NONE; + } CelSourceLocation policySourceLocation = celPolicySource.getOffsetLocation(policySourceOffset).orElse(null); if (policySourceLocation == null) { diff --git a/policy/src/main/java/dev/cel/policy/RuleComposer.java b/policy/src/main/java/dev/cel/policy/RuleComposer.java index 77a82b73c..732b60394 100644 --- a/policy/src/main/java/dev/cel/policy/RuleComposer.java +++ b/policy/src/main/java/dev/cel/policy/RuleComposer.java @@ -16,20 +16,25 @@ import static com.google.common.base.Preconditions.checkNotNull; import static com.google.common.collect.ImmutableList.toImmutableList; +import static java.util.stream.Collectors.toCollection; import com.google.auto.value.AutoValue; -import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import dev.cel.bundle.Cel; import dev.cel.common.CelAbstractSyntaxTree; import dev.cel.common.CelMutableAst; +import dev.cel.common.CelValidationException; import dev.cel.common.ast.CelConstant.Kind; import dev.cel.extensions.CelOptionalLibrary.Function; import dev.cel.optimizer.AstMutator; import dev.cel.optimizer.CelAstOptimizer; import dev.cel.parser.Operator; import dev.cel.policy.CelCompiledRule.CelCompiledMatch; +import dev.cel.policy.CelCompiledRule.CelCompiledMatch.OutputValue; import dev.cel.policy.CelCompiledRule.CelCompiledVariable; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; /** Package-private class for composing various rules into a single expression using optimizer. */ final class RuleComposer implements CelAstOptimizer { @@ -39,13 +44,8 @@ final class RuleComposer implements CelAstOptimizer { @Override public OptimizationResult optimize(CelAbstractSyntaxTree ast, Cel cel) { - RuleOptimizationResult result = optimizeRule(compiledRule); - return OptimizationResult.create( - result.ast().toParsedAst(), - compiledRule.variables().stream() - .map(CelCompiledVariable::celVarDecl) - .collect(toImmutableList()), - ImmutableList.of()); + RuleOptimizationResult result = optimizeRule(cel, compiledRule); + return OptimizationResult.create(result.ast().toParsedAst()); } @AutoValue @@ -59,9 +59,20 @@ static RuleOptimizationResult create(CelMutableAst ast, boolean isOptionalResult } } - private RuleOptimizationResult optimizeRule(CelCompiledRule compiledRule) { + private RuleOptimizationResult optimizeRule(Cel cel, CelCompiledRule compiledRule) { + cel = + cel.toCelBuilder() + .addVarDeclarations( + compiledRule.variables().stream() + .map(CelCompiledVariable::celVarDecl) + .collect(toImmutableList())) + .build(); + CelMutableAst matchAst = astMutator.newGlobalCall(Function.OPTIONAL_NONE.getFunction()); boolean isOptionalResult = true; + // Keep track of the last output ID that might cause type-check failure while attempting to + // compose the subgraphs. + long lastOutputId = 0; for (CelCompiledMatch match : Lists.reverse(compiledRule.matches())) { CelAbstractSyntaxTree conditionAst = match.condition(); boolean isTriviallyTrue = @@ -69,10 +80,12 @@ private RuleOptimizationResult optimizeRule(CelCompiledRule compiledRule) { && conditionAst.getExpr().constant().booleanValue(); switch (match.result().kind()) { case OUTPUT: - CelMutableAst outAst = CelMutableAst.fromCelAst(match.result().output()); + OutputValue matchOutput = match.result().output(); + CelMutableAst outAst = CelMutableAst.fromCelAst(matchOutput.ast()); if (isTriviallyTrue) { matchAst = outAst; isOptionalResult = false; + lastOutputId = matchOutput.id(); continue; } if (isOptionalResult) { @@ -85,9 +98,13 @@ private RuleOptimizationResult optimizeRule(CelCompiledRule compiledRule) { CelMutableAst.fromCelAst(conditionAst), outAst, matchAst); + assertComposedAstIsValid( + cel, matchAst, "conflicting output types found.", matchOutput.id(), lastOutputId); + lastOutputId = matchOutput.id(); continue; case RULE: - RuleOptimizationResult nestedRule = optimizeRule(match.result().rule()); + CelCompiledRule matchNestedRule = match.result().rule(); + RuleOptimizationResult nestedRule = optimizeRule(cel, matchNestedRule); CelMutableAst nestedRuleAst = nestedRule.ast(); if (isOptionalResult && !nestedRule.isOptionalResult()) { nestedRuleAst = @@ -101,6 +118,13 @@ private RuleOptimizationResult optimizeRule(CelCompiledRule compiledRule) { throw new IllegalArgumentException("Subrule early terminates policy"); } matchAst = astMutator.newMemberCall(nestedRuleAst, Function.OR.getFunction(), matchAst); + assertComposedAstIsValid( + cel, + matchAst, + String.format( + "failed composing the subrule '%s' due to conflicting output types.", + matchNestedRule.id().map(ValueString::value).orElse("")), + lastOutputId); break; } } @@ -127,9 +151,38 @@ static RuleComposer newInstance( return new RuleComposer(compiledRule, variablePrefix, iterationLimit); } + private void assertComposedAstIsValid( + Cel cel, CelMutableAst composedAst, String failureMessage, Long... ids) { + assertComposedAstIsValid(cel, composedAst, failureMessage, Arrays.asList(ids)); + } + + private void assertComposedAstIsValid( + Cel cel, CelMutableAst composedAst, String failureMessage, List ids) { + try { + cel.check(composedAst.toParsedAst()).getAst(); + } catch (CelValidationException e) { + ids = ids.stream().filter(id -> id > 0).collect(toCollection(ArrayList::new)); + throw new RuleCompositionException(failureMessage, e, ids); + } + } + private RuleComposer(CelCompiledRule compiledRule, String variablePrefix, int iterationLimit) { this.compiledRule = checkNotNull(compiledRule); this.variablePrefix = variablePrefix; this.astMutator = AstMutator.newInstance(iterationLimit); } + + static final class RuleCompositionException extends RuntimeException { + final String failureReason; + final List errorIds; + final CelValidationException compileException; + + private RuleCompositionException( + String failureReason, CelValidationException e, List errorIds) { + super(e); + this.failureReason = failureReason; + this.errorIds = errorIds; + this.compileException = e; + } + } } diff --git a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java index 22a18d69c..30915be4a 100644 --- a/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java +++ b/policy/src/test/java/dev/cel/policy/CelPolicyCompilerImplTest.java @@ -40,6 +40,7 @@ import dev.cel.policy.PolicyTestHelper.TestYamlPolicy; import dev.cel.runtime.CelRuntime.CelFunctionBinding; import dev.cel.testing.testdata.proto3.TestAllTypesProto.TestAllTypes; +import java.io.IOException; import java.util.Map; import java.util.Optional; import org.junit.Test; @@ -72,51 +73,22 @@ public void compileYamlPolicy_success(@TestParameter TestYamlPolicy yamlPolicy) } @Test - public void compileYamlPolicy_containsError_throws() throws Exception { + public void compileYamlPolicy_containsCompilationError_throws( + @TestParameter TestErrorYamlPolicy testCase) throws Exception { // Read config and produce an environment to compile policies - String configSource = readFromYaml("errors/config.yaml"); + String configSource = testCase.readConfigYamlContent(); CelPolicyConfig policyConfig = POLICY_CONFIG_PARSER.parse(configSource); Cel cel = policyConfig.extend(newCel(), CEL_OPTIONS); // Read the policy source - String policyFilePath = "errors/policy.yaml"; - String policySource = readFromYaml(policyFilePath); - CelPolicy policy = POLICY_PARSER.parse(policySource, policyFilePath); + String policySource = testCase.readPolicyYamlContent(); + CelPolicy policy = POLICY_PARSER.parse(policySource, testCase.getPolicyFilePath()); CelPolicyValidationException e = assertThrows( CelPolicyValidationException.class, () -> CelPolicyCompilerFactory.newPolicyCompiler(cel).build().compile(policy)); - assertThat(e) - .hasMessageThat() - .isEqualTo( - "ERROR: errors/policy.yaml:19:19: undeclared reference to 'spec' (in container '')\n" - + " | expression: spec.labels\n" - + " | ..................^\n" - + "ERROR: errors/policy.yaml:21:50: mismatched input 'resource' expecting {'=='," - + " '!=', 'in', '<', '<=', '>=', '>', '&&', '||', '[', '(', ')', '.', '-', '?'," - + " '+', '*', '/', '%%'}\n" - + " | expression: variables.want.filter(l, !(lin resource.labels))\n" - + " | .................................................^\n" - + "ERROR: errors/policy.yaml:21:66: extraneous input ')' expecting \n" - + " | expression: variables.want.filter(l, !(lin resource.labels))\n" - + " | .................................................................^\n" - + "ERROR: errors/policy.yaml:23:27: mismatched input '2' expecting {'}', ','}\n" - + " | expression: \"{1:305 2:569}\"\n" - + " | ..........................^\n" - + "ERROR: errors/policy.yaml:31:75: extraneous input ']' expecting ')'\n" - + " | \"missing one or more required labels:" - + " %s\".format(variables.missing])\n" - + " | ..........................................................................^\n" - + "ERROR: errors/policy.yaml:34:67: undeclared reference to 'format' (in container" - + " '')\n" - + " | \"invalid values provided on one or more labels:" - + " %s\".format([variables.invalid])\n" - + " | ..................................................................^\n" - + "ERROR: errors/policy.yaml:35:24: found no matching overload for '_==_' applied" - + " to '(bool, string)' (candidates: (%A0, %A0))\n" - + " | - condition: false == \"0\"\n" - + " | .......................^"); + assertThat(e).hasMessageThat().isEqualTo(testCase.readExpectedErrorsBaseline()); } @Test @@ -332,4 +304,34 @@ private enum MultilineErrorTest { this.expected = expected; } } + + private enum TestErrorYamlPolicy { + COMPILE_ERRORS("compile_errors"), + COMPOSE_ERRORS_CONFLICTING_OUTPUT("compose_errors_conflicting_output"), + COMPOSE_ERRORS_CONFLICTING_SUBRULE("compose_errors_conflicting_subrule"); + + private final String name; + private final String policyFilePath; + + private String getPolicyFilePath() { + return policyFilePath; + } + + private String readPolicyYamlContent() throws IOException { + return readFromYaml(String.format("%s/policy.yaml", name)); + } + + private String readConfigYamlContent() throws IOException { + return readFromYaml(String.format("%s/config.yaml", name)); + } + + private String readExpectedErrorsBaseline() throws IOException { + return readFromYaml(String.format("%s/expected_errors.baseline", name)); + } + + TestErrorYamlPolicy(String name) { + this.name = name; + this.policyFilePath = String.format("%s/policy.yaml", name); + } + } } diff --git a/policy/src/test/resources/BUILD.bazel b/policy/src/test/resources/BUILD.bazel index 71285b61a..4876f22fb 100644 --- a/policy/src/test/resources/BUILD.bazel +++ b/policy/src/test/resources/BUILD.bazel @@ -10,5 +10,5 @@ package( filegroup( name = "policy_yaml_files", - srcs = glob(["**/*.yaml"]), + srcs = glob(["**/*.yaml"]) + glob(["**/*.baseline"]), ) diff --git a/policy/src/test/resources/errors/config.yaml b/policy/src/test/resources/compile_errors/config.yaml similarity index 100% rename from policy/src/test/resources/errors/config.yaml rename to policy/src/test/resources/compile_errors/config.yaml diff --git a/policy/src/test/resources/compile_errors/expected_errors.baseline b/policy/src/test/resources/compile_errors/expected_errors.baseline new file mode 100644 index 000000000..a8c0ec047 --- /dev/null +++ b/policy/src/test/resources/compile_errors/expected_errors.baseline @@ -0,0 +1,24 @@ +ERROR: compile_errors/policy.yaml:19:19: undeclared reference to 'spec' (in container '') + | expression: spec.labels + | ..................^ +ERROR: compile_errors/policy.yaml:21:50: mismatched input 'resource' expecting {'==', '!=', 'in', '<', '<=', '>=', '>', '&&', '||', '[', '(', ')', '.', '-', '?', '+', '*', '/', '%%'} + | expression: variables.want.filter(l, !(lin resource.labels)) + | .................................................^ +ERROR: compile_errors/policy.yaml:21:66: extraneous input ')' expecting + | expression: variables.want.filter(l, !(lin resource.labels)) + | .................................................................^ +ERROR: compile_errors/policy.yaml:23:27: mismatched input '2' expecting {'}', ','} + | expression: "{1:305 2:569}" + | ..........................^ +ERROR: compile_errors/policy.yaml:31:75: extraneous input ']' expecting ')' + | "missing one or more required labels: %s".format(variables.missing]) + | ..........................................................................^ +ERROR: compile_errors/policy.yaml:34:67: undeclared reference to 'format' (in container '') + | "invalid values provided on one or more labels: %s".format([variables.invalid]) + | ..................................................................^ +ERROR: compile_errors/policy.yaml:35:19: condition must produce a boolean output. + | - condition: '1' + | ..................^ +ERROR: compile_errors/policy.yaml:38:24: found no matching overload for '_==_' applied to '(bool, string)' (candidates: (%A0, %A0)) + | - condition: false == "0" + | .......................^ \ No newline at end of file diff --git a/policy/src/test/resources/errors/policy.yaml b/policy/src/test/resources/compile_errors/policy.yaml similarity index 92% rename from policy/src/test/resources/errors/policy.yaml rename to policy/src/test/resources/compile_errors/policy.yaml index e322e8a83..c69bda507 100644 --- a/policy/src/test/resources/errors/policy.yaml +++ b/policy/src/test/resources/compile_errors/policy.yaml @@ -32,6 +32,9 @@ rule: - condition: variables.invalid.size() > 0 output: | "invalid values provided on one or more labels: %s".format([variables.invalid]) + - condition: '1' + output: | + "condition wrong type" - condition: false == "0" output: | - "wrong type" + "condition type-check failure" diff --git a/policy/src/test/resources/compose_errors_conflicting_output/config.yaml b/policy/src/test/resources/compose_errors_conflicting_output/config.yaml new file mode 100644 index 000000000..5d048a225 --- /dev/null +++ b/policy/src/test/resources/compose_errors_conflicting_output/config.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "labels" +variables: +- name: "resource" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "dyn" diff --git a/policy/src/test/resources/compose_errors_conflicting_output/expected_errors.baseline b/policy/src/test/resources/compose_errors_conflicting_output/expected_errors.baseline new file mode 100644 index 000000000..3e2624b64 --- /dev/null +++ b/policy/src/test/resources/compose_errors_conflicting_output/expected_errors.baseline @@ -0,0 +1,6 @@ +ERROR: compose_errors_conflicting_output/policy.yaml:22:14: conflicting output types found. + | output: "false" + | .............^ +ERROR: compose_errors_conflicting_output/policy.yaml:23:14: conflicting output types found. + | - output: "{'banned': true}" + | .............^ \ No newline at end of file diff --git a/policy/src/test/resources/compose_errors_conflicting_output/policy.yaml b/policy/src/test/resources/compose_errors_conflicting_output/policy.yaml new file mode 100644 index 000000000..a5ed5c09c --- /dev/null +++ b/policy/src/test/resources/compose_errors_conflicting_output/policy.yaml @@ -0,0 +1,23 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: nested_rule +rule: + variables: + - name: "permitted_regions" + expression: "['us', 'uk', 'es']" + match: + - condition: resource.origin in variables.permitted_regions + output: "false" + - output: "{'banned': true}" diff --git a/policy/src/test/resources/compose_errors_conflicting_subrule/config.yaml b/policy/src/test/resources/compose_errors_conflicting_subrule/config.yaml new file mode 100644 index 000000000..5d048a225 --- /dev/null +++ b/policy/src/test/resources/compose_errors_conflicting_subrule/config.yaml @@ -0,0 +1,22 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: "labels" +variables: +- name: "resource" + type: + type_name: "map" + params: + - type_name: "string" + - type_name: "dyn" diff --git a/policy/src/test/resources/compose_errors_conflicting_subrule/expected_errors.baseline b/policy/src/test/resources/compose_errors_conflicting_subrule/expected_errors.baseline new file mode 100644 index 000000000..559d62e1d --- /dev/null +++ b/policy/src/test/resources/compose_errors_conflicting_subrule/expected_errors.baseline @@ -0,0 +1,3 @@ +ERROR: compose_errors_conflicting_subrule/policy.yaml:36:14: failed composing the subrule 'banned regions' due to conflicting output types. + | output: "{'banned': false}" + | .............^ \ No newline at end of file diff --git a/policy/src/test/resources/compose_errors_conflicting_subrule/policy.yaml b/policy/src/test/resources/compose_errors_conflicting_subrule/policy.yaml new file mode 100644 index 000000000..9df1df8d0 --- /dev/null +++ b/policy/src/test/resources/compose_errors_conflicting_subrule/policy.yaml @@ -0,0 +1,37 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: nested_rule +rule: + variables: + - name: "permitted_regions" + expression: "['us', 'uk', 'es']" + match: + - rule: + id: "banned regions" + description: > + determine whether the resource origin is in the banned + list. If the region is also in the permitted list, the + ban has no effect. + variables: + - name: "banned_regions" + expression: "{'us': false, 'ru': false, 'ir': false}" + match: + - condition: | + resource.origin in variables.banned_regions && + !(resource.origin in variables.permitted_regions) + output: "true" + - condition: resource.origin in variables.permitted_regions + output: "{'banned': false}" + - output: "{'banned': true}" From d82820cb03d6100b45828870e82b8d6a4eb9028d Mon Sep 17 00:00:00 2001 From: Sokwhan Huh Date: Mon, 15 Jul 2024 17:42:44 -0700 Subject: [PATCH 33/33] Release 0.6.0 PiperOrigin-RevId: 652649980 --- README.md | 4 ++-- publish/cel_version.bzl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index b9cc5f3a9..b61d940f9 100644 --- a/README.md +++ b/README.md @@ -55,14 +55,14 @@ CEL-Java is available in Maven Central Repository. [Download the JARs here][8] o dev.cel cel - 0.5.2 + 0.6.0 ``` **Gradle** ```gradle -implementation 'dev.cel:cel:0.5.2' +implementation 'dev.cel:cel:0.6.0' ``` Then run this example: diff --git a/publish/cel_version.bzl b/publish/cel_version.bzl index 866840e0c..96be40f81 100644 --- a/publish/cel_version.bzl +++ b/publish/cel_version.bzl @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. """Maven artifact version for CEL.""" -CEL_VERSION = "0.5.2" +CEL_VERSION = "0.6.0"