Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions verifier/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,18 @@ by the Z3 solver during its search phase.
If the solver times out, the verifier will throw a `CelVerificationException`
which should be explicitly caught and handled by the caller.

### Validating Counterexamples with the CEL Runtime

`enableCegar(true)` checks the inputs the solver finds with the CEL
runtime of the verifier's environment: a confirmed `INCONCLUSIVE` counterexample
(or satisfying input) becomes `VIOLATED` (or `VERIFIED`), and any refuted one
becomes `INCONCLUSIVE`. Only exact inputs are checked, when no unknown
identifiers are registered; when the runtime can't evaluate them (e.g. a
function has no binding), the message says why. Policy invariants aren't
checked yet. The runtime must follow the CEL spec (e.g. `CelOptions.DEFAULT`, which has no
iteration limit), and a function with bindings must have one for each overload,
which is synchronous, deterministic and doesn't throw.

### Comprehensions and Bounded Model Checking

Because CEL lists/maps can be dynamically sized, the verifier cannot
Expand Down
7 changes: 7 additions & 0 deletions verifier/src/main/java/dev/cel/verifier/BUILD.bazel
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,7 @@ java_library(
"CelZ3StructTranslator.java",
"CelZ3TranslationContext.java",
"CelZ3TypeConstraintGenerator.java",
"RuntimeOracle.java",
"TranslatedValue.java",
],
compatible_with = [],
Expand All @@ -184,10 +185,12 @@ java_library(
"//bundle:cel",
"//common:cel_ast",
"//common:compiler_common",
"//common:error_codes",
"//common:operator",
"//common/ast",
"//common/ast:cel_block",
"//common/internal:cel_descriptor_pools",
"//common/internal:map_keys",
"//common/internal:proto_time_utils",
"//common/internal:well_known_proto",
"//common/types",
Expand All @@ -198,6 +201,10 @@ java_library(
"//optimizer",
"//optimizer:optimization_exception",
"//optimizer:optimizer_builder",
"//runtime",
"//runtime:function_resolver",
"//runtime:resolved_overload",
"//runtime:unknown_attributes",
"//verifier/axioms",
"@maven//:com_google_errorprone_error_prone_annotations",
"@maven//:com_google_guava_guava",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,7 @@ private TranslatedValue translateIdent(CelExpr celExpr, CelAbstractSyntaxTree as
}
String name = ident.name();
CelType type = ast.getTypeOrThrow(exprId);
if (type instanceof TypeType) {
if (type instanceof TypeType && !context.lookupSymbol(name).isPresent()) {
TypeType typeType = (TypeType) type;
if (typeType.type().kind() != CelKind.DYN) {
return TranslatedValue.create(
Expand Down
17 changes: 17 additions & 0 deletions verifier/src/main/java/dev/cel/verifier/CelVerifierBuilder.java
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,23 @@ public interface CelVerifierBuilder {
@CanIgnoreReturnValue
CelVerifierBuilder setComprehensionUnrollLimit(int unrollLimit);

/**
* Checks the inputs the solver finds by evaluating them with the CEL runtime of the verifier's
* environment (off by default). For now this only validates them, without refining the solver
* (counterexample-guided abstraction refinement, CEGAR). An {@code INCONCLUSIVE} counterexample
* or satisfying input that the runtime confirms becomes {@code VIOLATED} (or {@code VERIFIED} for
* satisfiability), and any counterexample or satisfying input it disagrees with becomes {@code
* INCONCLUSIVE}.
*
* <p>Inputs are checked only when all their values are exact and no unknown identifiers are
* registered. When the runtime can't evaluate them, e.g. a function has no binding, the message
* says why. Policy invariants aren't checked yet. The runtime must follow the CEL spec (e.g.
* {@code CelOptions.DEFAULT}, which has no iteration limit), and a function with bindings must
* have one for each overload, which is synchronous, deterministic and doesn't throw.
*/
@CanIgnoreReturnValue
CelVerifierBuilder enableCegar(boolean value);

/** Builds the {@link CelVerifier} instance. */
CelVerifier build();
}
Expand Down
126 changes: 111 additions & 15 deletions verifier/src/main/java/dev/cel/verifier/CelVerifierZ3Impl.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import static com.google.common.base.Preconditions.checkNotNull;
import static com.google.common.collect.ImmutableList.toImmutableList;

import com.google.auto.value.AutoValue;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
Expand All @@ -41,12 +42,15 @@
import dev.cel.optimizer.CelOptimizationException;
import dev.cel.optimizer.CelOptimizer;
import dev.cel.optimizer.CelOptimizerFactory;
import dev.cel.runtime.CelEvaluationException;
import dev.cel.verifier.axioms.CelZ3FunctionAxiom;
import dev.cel.verifier.axioms.CelZ3StandardAxioms;
import java.time.Duration;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.function.Predicate;

/** Z3 implementation of the CelVerifier. */
@Immutable
Expand Down Expand Up @@ -91,6 +95,7 @@ public Optional<CelType> findType(String typeName) {
private final ImmutableSet<String> unknownIdentifiers;
private final CelZ3FunctionRegistry functionRegistry;
private final CelTypeProvider typeProvider;
private final boolean enableCegar;

@SuppressWarnings("Immutable") // Cel environment is immutable, just not marked as such
private final Cel cel;
Expand All @@ -111,6 +116,7 @@ static final class Builder implements CelVerifierBuilder {
private final ImmutableList.Builder<CelZ3FunctionAxiom> functionAxioms;
private final Cel cel;
private CelTypeProvider typeProvider;
private boolean enableCegar;

@Override
@CanIgnoreReturnValue
Expand Down Expand Up @@ -144,6 +150,13 @@ public CelVerifierBuilder setComprehensionUnrollLimit(int unrollLimit) {
return this;
}

@Override
@CanIgnoreReturnValue
public CelVerifierBuilder enableCegar(boolean value) {
this.enableCegar = value;
return this;
}

@CanIgnoreReturnValue
Builder addFunctionAxioms(CelZ3FunctionAxiom... axioms) {
return addFunctionAxioms(Arrays.asList(axioms));
Expand All @@ -170,6 +183,7 @@ public CelVerifier build() {
unknownIdentifiers.build(),
registry,
typeProvider,
enableCegar,
cel);
}

Expand Down Expand Up @@ -208,17 +222,19 @@ public CelVerificationResult verifyEquivalence(
CanonicalizationOptimizer.newInstance(
CanonicalizationOptimizer.CanonicalizationOptions.newBuilder().build()))
.build();
CelAbstractSyntaxTree canonicalA = astA;
CelAbstractSyntaxTree canonicalB = astB;
try {
astA = optimizer.optimize(astA);
astB = optimizer.optimize(astB);
canonicalA = optimizer.optimize(astA);
canonicalB = optimizer.optimize(astB);
} catch (CelOptimizationException e) {
// Fall back to original ASTs if canonicalization or re-typechecking fails
}
try (Context ctx = new Context(ImmutableMap.of("model", "true"))) {
CelAstToZ3Translator translator = newTranslator(ctx, /* parameterizeUnknowns= */ true);

TranslatedValue tvA = translator.translate(astA);
TranslatedValue tvB = translator.translate(astB);
TranslatedValue tvA = translator.translate(canonicalA);
TranslatedValue tvB = translator.translate(canonicalB);

BoolExpr divergenceCondition = ctx.mkNot(ctx.mkEq(tvA.z3Expr(), tvB.z3Expr()));
// Parameterized unknowns are keyed on the operation producing them, whereas CEL identifies
Expand All @@ -243,7 +259,8 @@ public CelVerificationResult verifyEquivalence(
/* unknownCondition= */ ctx.mkFalse(),
/* isCounterexample= */ true,
/* candidate= */ "a divergence",
/* exactMatchMessage= */ "Equivalence violation detected.");
/* exactMatchMessage= */ "Equivalence violation detected.",
runtimeCheck(results -> !results.get(0).equals(results.get(1)), astA, astB));
}
}

Expand Down Expand Up @@ -295,7 +312,8 @@ CelVerificationResult verifyImplication(
unknownCondition,
/* isCounterexample= */ true,
/* candidate= */ "a counterexample",
/* exactMatchMessage= */ String.format("%s violation detected.", subjectName));
/* exactMatchMessage= */ String.format("%s violation detected.", subjectName),
/* runtimeCheck= */ RuntimeChecker.UNCHECKED);
}
}

Expand All @@ -316,7 +334,8 @@ private CelVerificationResult checkSatisfiability(
/* unknownCondition= */ ctx.mkFalse(),
/* isCounterexample= */ true,
/* candidate= */ "a counterexample",
/* exactMatchMessage= */ "Condition is not always true.");
/* exactMatchMessage= */ "Condition is not always true.",
runtimeCheck(results -> !results.get(0).equals(true), ast));
}
BoolExpr isUnmodeledUnknown = ctx.mkAnd(tv.isZ3Unknown(), tv.isApproximate());
return search(
Expand All @@ -327,13 +346,16 @@ private CelVerificationResult checkSatisfiability(
isUnmodeledUnknown,
/* isCounterexample= */ false,
/* candidate= */ "a satisfying input",
/* exactMatchMessage= */ "Condition is satisfiable.");
/* exactMatchMessage= */ "Condition is satisfiable.",
runtimeCheck(results -> results.get(0).equals(true), ast));
}
}

/**
* Searches for a {@code condition} match: first an exact one, then an approximate one, then an
* unmodeled result satisfying {@code unknownCondition} that may stand in for one.
* unmodeled result satisfying {@code unknownCondition} that may stand in for one. {@code
* runtimeCheck} checks a witness with the runtime, noting why if it couldn't. Confirming an
* approximate match doesn't count internal runtime errors.
*/
private CelVerificationResult search(
Context ctx,
Expand All @@ -343,7 +365,8 @@ private CelVerificationResult search(
BoolExpr unknownCondition,
boolean isCounterexample,
String candidate,
String exactMatchMessage)
String exactMatchMessage,
RuntimeChecker runtimeCheck)
throws CelVerificationException {
Solver solver = newSolver(ctx);
solver.add(translator.getTypeConstraints().toArray(new BoolExpr[0]));
Expand All @@ -353,9 +376,14 @@ private CelVerificationResult search(
Status status = checkSat(solver, typeSystem, condition, ctx.mkNot(taint));
if (status == Status.SATISFIABLE) {
CelWitness witness = extractWitness(ctx, translator, solver.getModel());
return isCounterexample
? CelVerificationResult.failed(exactMatchMessage, witness)
: CelVerificationResult.verified(exactMatchMessage, witness);
RuntimeCheck check = runtimeCheck.check(witness, /* countInternalErrors= */ true);
if (!check.isMatch().orElse(true)) {
return CelVerificationResult.inconclusive(
"Inconclusive: the CEL runtime disagrees with " + candidate + " the verifier found.",
witness,
isCounterexample);
}
return match(isCounterexample, exactMatchMessage + check.note(), witness);
} else if (status == Status.UNKNOWN) {
// Depends on solver internals; not deterministically testable.
return solverGaveUp(solver);
Expand All @@ -365,9 +393,19 @@ private CelVerificationResult search(
status = checkSat(solver, typeSystem, condition);
if (status == Status.SATISFIABLE) {
Model model = solver.getModel();
CelWitness witness = extractWitness(ctx, translator, model);
RuntimeCheck check = runtimeCheck.check(witness, /* countInternalErrors= */ false);
if (check.isMatch().orElse(false)) {
return match(
isCounterexample,
String.format(
"%s The CEL runtime confirmed %s the verifier approximated.",
exactMatchMessage, candidate),
witness);
}
return CelVerificationResult.inconclusive(
inconclusiveReason(candidate, translator, model, taint),
extractWitness(ctx, translator, model),
inconclusiveReason(candidate, translator, model, taint) + check.note(),
witness,
isCounterexample);
} else if (status == Status.UNKNOWN) {
// Depends on solver internals; not deterministically testable.
Expand All @@ -389,6 +427,62 @@ private CelVerificationResult search(
: CelVerificationResult.failed("Condition is not satisfiable.");
}

private static CelVerificationResult match(
boolean isCounterexample, String reason, CelWitness witness) {
return isCounterexample
? CelVerificationResult.failed(reason, witness)
: CelVerificationResult.verified(reason, witness);
}

private interface RuntimeChecker {
RuntimeChecker UNCHECKED = (witness, countInternalErrors) -> RuntimeCheck.UNCHECKED;

RuntimeCheck check(CelWitness witness, boolean countInternalErrors);
}

// Whether the runtime agrees a witness is a match, if it checked it, and a note for the message.
@AutoValue
abstract static class RuntimeCheck {
static final RuntimeCheck UNCHECKED =
new AutoValue_CelVerifierZ3Impl_RuntimeCheck(Optional.empty(), "");

abstract Optional<Boolean> isMatch();

abstract String note();

static RuntimeCheck of(boolean isMatch) {
return new AutoValue_CelVerifierZ3Impl_RuntimeCheck(
Optional.of(isMatch), isMatch ? "" : " The CEL runtime disagrees with the one found.");
}

static RuntimeCheck failed(String reason) {
return new AutoValue_CelVerifierZ3Impl_RuntimeCheck(
Optional.empty(),
" The CEL runtime couldn't check it: " + reason + (reason.endsWith(".") ? "" : "."));
}
}

// Checks the witness's inputs with the runtime, if refinement is on and they're exact. Unknowns
// aren't inputs the runtime can take.
private RuntimeChecker runtimeCheck(
Predicate<List<Object>> isMatch, CelAbstractSyntaxTree... asts) {
if (!enableCegar || !unknownIdentifiers.isEmpty()) {
return RuntimeChecker.UNCHECKED;
}
return (witness, countInternalErrors) -> {
Optional<ImmutableMap<String, Object>> inputs = RuntimeOracle.exactInputs(witness);
if (!inputs.isPresent()) {
return RuntimeCheck.UNCHECKED;
}
try {
return RuntimeCheck.of(
isMatch.test(RuntimeOracle.evaluate(cel, inputs.get(), countInternalErrors, asts)));
} catch (CelEvaluationException e) {
return RuntimeCheck.failed(e.getMessage());
}
};
}

// Throws on a timeout. Leaves the assertions in place unless they're unsatisfiable, so the model
// or the reason the solver gave up can be read.
private static Status checkSat(Solver solver, CelZ3TypeSystem typeSystem, BoolExpr... assertions)
Expand Down Expand Up @@ -468,12 +562,14 @@ private CelVerifierZ3Impl(
ImmutableSet<String> unknownIdentifiers,
CelZ3FunctionRegistry functionRegistry,
CelTypeProvider typeProvider,
boolean enableCegar,
Cel cel) {
this.timeout = timeout;
this.comprehensionUnrollLimit = comprehensionUnrollLimit;
this.unknownIdentifiers = unknownIdentifiers;
this.functionRegistry = functionRegistry;
this.typeProvider = typeProvider;
this.enableCegar = enableCegar;
this.cel = cel;
}
}
Loading
Loading