[SYSTEMDS-3972] Merge PR Differential privacy for aggregates

.github/workflows/javaTests.yml

@@ -83,6 +83,7 @@ jobs:
           "**.functions.frame.**,**.functions.indexing.**,**.functions.io.**,**.functions.iogen.**",
           "**.functions.dnn.**,**.functions.ooc.**",
           "**.functions.paramserv.**",
+          "**.functions.privacy.**",
           "**.functions.recompile.**,**.functions.misc.**",
           "**.functions.mlcontext.**",
           "**.functions.nary.**,**.functions.quaternary.**",

src/main/java/org/apache/sysds/common/Builtins.java

@@ -127,6 +127,7 @@ public enum Builtins {
 	SETDIFF("setdiff", true),
 	DIST("dist", true),
 	DMV("dmv", true),
+	DP_SET_BUDGET("dp_set_budget", false),
 	DROP_INVALID_TYPE("dropInvalidType", false),
 	DROP_INVALID_LENGTH("dropInvalidLength", false),
 	EIGEN("eigen", false, ReturnType.MULTI_RETURN),
@@ -377,6 +378,8 @@ public enum Builtins {
 	COUNT_DISTINCT_APPROX_ROW("rowCountDistinctApprox", false, true),
 	COUNT_DISTINCT_APPROX_COL("colCountDistinctApprox", false, true),
 	CVLM("cvlm", true, false),
+	DP_GAUSSIAN("dp_gaussian", false, true),
+	DP_LAPLACE("dp_laplace", false, true),
 	GROUPEDAGG("aggregate", "groupedAggregate", false, true),
 	INVCDF("icdf", false, true),
 	LISTNV("list", false, true), //note: builtin and parbuiltin

src/main/java/org/apache/sysds/common/Opcodes.java

@@ -196,24 +196,26 @@ public enum Opcodes {
 
 	//Parametrized builtin functions
 	AUTODIFF("autoDiff", InstructionType.ParameterizedBuiltin),
-	CONTAINS("contains", InstructionType.ParameterizedBuiltin),
-	PARAMSERV("paramserv", InstructionType.ParameterizedBuiltin),
-	NVLIST("nvlist", InstructionType.ParameterizedBuiltin),
 	CDF("cdf", InstructionType.ParameterizedBuiltin),
-	INVCDF("invcdf", InstructionType.ParameterizedBuiltin),
+	CONTAINS("contains", InstructionType.ParameterizedBuiltin),
+	DP_GAUSSIAN("dp_gaussian", InstructionType.ParameterizedBuiltin),
+	DP_LAPLACE("dp_laplace", InstructionType.ParameterizedBuiltin),
 	GROUPEDAGG("groupedagg", InstructionType.ParameterizedBuiltin),
-	RMEMPTY("rmempty", InstructionType.ParameterizedBuiltin),
-	REPLACE("replace", InstructionType.ParameterizedBuiltin),
+	INVCDF("invcdf", InstructionType.ParameterizedBuiltin),
 	LOWERTRI("lowertri", InstructionType.ParameterizedBuiltin),
-	UPPERTRI("uppertri", InstructionType.ParameterizedBuiltin),
+	NVLIST("nvlist", InstructionType.ParameterizedBuiltin),
+	PARAMSERV("paramserv", InstructionType.ParameterizedBuiltin),
+	REPLACE("replace", InstructionType.ParameterizedBuiltin),
 	REXPAND("rexpand", InstructionType.ParameterizedBuiltin),
-	TOSTRING("toString", InstructionType.ParameterizedBuiltin),
+	RMEMPTY("rmempty", InstructionType.ParameterizedBuiltin),
 	TOKENIZE("tokenize", InstructionType.ParameterizedBuiltin),
+	TOSTRING("toString", InstructionType.ParameterizedBuiltin),
 	TRANSFORMAPPLY("transformapply", InstructionType.ParameterizedBuiltin),
 	TRANSFORMDECODE("transformdecode", InstructionType.ParameterizedBuiltin),
 	TRANSFORMCOLMAP("transformcolmap", InstructionType.ParameterizedBuiltin),
 	TRANSFORMMETA("transformmeta", InstructionType.ParameterizedBuiltin),
 	TRANSFORMENCODE("transformencode", InstructionType.MultiReturnParameterizedBuiltin,  InstructionType.MultiReturnBuiltin),
+	UPPERTRI("uppertri", InstructionType.ParameterizedBuiltin),
 
 	GET_CATEGORICAL_MASK("get_categorical_mask", InstructionType.Binary),
 

src/main/java/org/apache/sysds/common/Types.java

@@ -806,10 +806,9 @@ public static ReOrgOp valueOfByOpcode(String opcode) {
 
 	/** Parameterized operations that require named variable arguments */
 	public enum ParamBuiltinOp {
-		AUTODIFF, CDF, CONTAINS, INVALID, INVCDF, GROUPEDAGG, RMEMPTY, REPLACE, REXPAND,
-		LOWER_TRI, UPPER_TRI,
-		TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMCOLMAP, TRANSFORMMETA,
-		TOKENIZE, TOSTRING, LIST, PARAMSERV
+		AUTODIFF, CDF, CONTAINS, DP_GAUSSIAN, DP_LAPLACE, INVALID, INVCDF, GROUPEDAGG, RMEMPTY, REPLACE, REXPAND,
+		LOWER_TRI, UPPER_TRI, TRANSFORMAPPLY, TRANSFORMDECODE, TRANSFORMCOLMAP, TRANSFORMMETA, TOKENIZE, TOSTRING, LIST,
+		PARAMSERV
 	}
 
 	/** Deep Neural Network specific operations */

src/main/java/org/apache/sysds/hops/ParameterizedBuiltinOp.java

@@ -182,7 +182,7 @@ public Lop constructLops()
 			}
 			case CONTAINS:
 			case CDF:
-			case INVCDF: 
+			case INVCDF:
 			case REPLACE:
 			case LOWER_TRI:
 			case UPPER_TRI:
@@ -194,10 +194,12 @@ public Lop constructLops()
 			case TOSTRING:
 			case PARAMSERV:
 			case LIST:
-			case AUTODIFF:{
+			case AUTODIFF:
+			case DP_GAUSSIAN:
+			case DP_LAPLACE: {
 				ParameterizedBuiltin pbilop = new ParameterizedBuiltin(
 					inputlops, _op, getDataType(), getValueType(), et);
-				if( isMultiThreadedOpType() )
+				if(isMultiThreadedOpType())
 					pbilop.setNumThreads(OptimizerUtils.getConstrainedNumThreads(_maxNumThreads));
 				setOutputDimensions(pbilop);
 				setLineNumbers(pbilop);
@@ -688,7 +690,17 @@ else if( _op == ParamBuiltinOp.TRANSFORMAPPLY ) {
 				return new MatrixCharacteristics(dc.getRows(), dc.getCols(), -1, dc.getLength());
 			}
 		}
-
+		else if(_op == ParamBuiltinOp.DP_GAUSSIAN || _op == ParamBuiltinOp.DP_LAPLACE) {
+			if(dc.dimsKnown()) {
+				Hop query = getParameterHop("query");
+				String queryVal = (query instanceof LiteralOp) ? ((LiteralOp) query).getStringValue() : null;
+				if("colMeans".equals(queryVal) || "colSums".equals(queryVal))
+					ret = new MatrixCharacteristics(1, dc.getCols(), -1, dc.getCols());
+				else if("identity".equals(queryVal))
+					ret = new MatrixCharacteristics(dc.getRows(), dc.getCols(), -1, dc.getLength());
+			}
+		}
+
 		return ret;
 	}
 	@Override
@@ -755,10 +767,10 @@ && getTargetHop().areDimsBelowThreshold() ) {
 		// 2. For paramserv function, always be CP mode so that
 		// the parameter server could have a central instruction
 		// to determine the local or remote workers
-		if (_op == ParamBuiltinOp.TRANSFORMCOLMAP || _op == ParamBuiltinOp.TRANSFORMMETA
-				|| _op == ParamBuiltinOp.TOSTRING || _op == ParamBuiltinOp.LIST
-				|| _op == ParamBuiltinOp.CDF || _op == ParamBuiltinOp.INVCDF
-				|| _op == ParamBuiltinOp.PARAMSERV) {
+		if(_op == ParamBuiltinOp.TRANSFORMCOLMAP || _op == ParamBuiltinOp.TRANSFORMMETA ||
+			_op == ParamBuiltinOp.TOSTRING || _op == ParamBuiltinOp.LIST || _op == ParamBuiltinOp.CDF ||
+			_op == ParamBuiltinOp.INVCDF || _op == ParamBuiltinOp.PARAMSERV || _op == ParamBuiltinOp.DP_GAUSSIAN ||
+			_op == ParamBuiltinOp.DP_LAPLACE) {
 			_etype = ExecType.CP;
 		}
 
@@ -940,17 +952,19 @@ public boolean compare( Hop that )
 	{
 		if( !(that instanceof ParameterizedBuiltinOp) )
 			return false;
-
-		ParameterizedBuiltinOp that2 = (ParameterizedBuiltinOp)that;	
-		boolean ret = (_op == that2._op
-					  && _paramIndexMap!=null && that2._paramIndexMap!=null
-					  && _paramIndexMap.size() == that2._paramIndexMap.size()
-					  && _outputEmptyBlocks == that2._outputEmptyBlocks
-					  && _outputPermutationMatrix == that2._outputPermutationMatrix );
-		if( ret )
-		{
-			for( Entry<String,Integer> e : _paramIndexMap.entrySet() )
-			{
+
+		// NOTE: dp_gaussian, dp_laplace draw fresh random noise on every call and record a
+		// privacy-budget charge as a side effect (see DPBuiltinOps.release), so two
+		// syntactically identical calls must never be merged into one execution.
+		if(_op == ParamBuiltinOp.DP_GAUSSIAN || _op == ParamBuiltinOp.DP_LAPLACE)
+			return false;
+
+		ParameterizedBuiltinOp that2 = (ParameterizedBuiltinOp) that;
+		boolean ret = (_op == that2._op && _paramIndexMap != null && that2._paramIndexMap != null &&
+			_paramIndexMap.size() == that2._paramIndexMap.size() && _outputEmptyBlocks == that2._outputEmptyBlocks &&
+			_outputPermutationMatrix == that2._outputPermutationMatrix);
+		if(ret) {
+			for(Entry<String, Integer> e : _paramIndexMap.entrySet()) {
 				String key1 = e.getKey();
 				int pos1 = e.getValue();
 				int pos2 = that2._paramIndexMap.get(key1);

src/main/java/org/apache/sysds/lops/ParameterizedBuiltin.java

@@ -204,7 +204,19 @@ public String getInstructions(String output)
 				compileGenericParamMap(sb, _inputParams);
 				break;
 			}
-
+			case DP_GAUSSIAN: {
+				sb.append(Opcodes.DP_GAUSSIAN);
+				sb.append(OPERAND_DELIMITOR);
+				compileGenericParamMap(sb, _inputParams);
+				break;
+			}
+			case DP_LAPLACE: {
+				sb.append(Opcodes.DP_LAPLACE);
+				sb.append(OPERAND_DELIMITOR);
+				compileGenericParamMap(sb, _inputParams);
+				break;
+			}
+
 			default:
 				throw new LopsException(this.printErrorLocation() + "In ParameterizedBuiltin Lop, Unknown operation: " + _operation);
 		}

src/main/java/org/apache/sysds/parser/BuiltinFunctionExpression.java

@@ -1995,16 +1995,40 @@ else if(this.getOpCode() == Builtins.MAX_POOL || this.getOpCode() == Builtins.AV
 			}
 			break;
 		case LOCAL:
-			if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_LOCAL_COMMAND){
+			if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_LOCAL_COMMAND) {
 				checkNumParameters(1);
 				checkMatrixParam(getFirstExpr());
 				output.setDataType(DataType.MATRIX);
 				output.setDimensions(id.getDim1(), id.getDim2());
 				output.setBlocksize (id.getBlocksize());
 				output.setValueType(id.getValueType());
 			}
-			else 
+			else {
 				raiseValidateError("Local instruction not allowed in dml script");
+			}
+			break;
+		case DP_SET_BUDGET: {
+			checkNumParameters(2);
+			checkScalarParam(getFirstExpr());
+			checkScalarParam(getSecondExpr());
+			// resolved entirely at compile time (see DMLTranslator), so both
+			// arguments must be known before the HOP DAG is built.
+			if (!isConstant(getFirstExpr()) || !isConstant(getSecondExpr()))
+				raiseValidateError(getOpCode() + ": 'epsilon' and 'delta' must be compile-time numeric literals",
+					false, LanguageErrorCodes.INVALID_PARAMETERS);
+			double dpSetBudgetEpsilon = getDoubleValue(getFirstExpr());
+			double dpSetBudgetDelta = getDoubleValue(getSecondExpr());
+			if (dpSetBudgetEpsilon <= 0)
+				raiseValidateError(getOpCode() + ": epsilon must be > 0, got " + dpSetBudgetEpsilon,
+					false, LanguageErrorCodes.INVALID_PARAMETERS);
+			if ((dpSetBudgetDelta <= 0) || (dpSetBudgetDelta >= 1))
+				raiseValidateError(getOpCode() + ": delta must be in (0,1), got " + dpSetBudgetDelta,
+					false, LanguageErrorCodes.INVALID_PARAMETERS);
+			output.setDataType(DataType.SCALAR);
+			output.setValueType(ValueType.FP64);
+			output.setDimensions(0, 0);
+			break;
+		}
 		case COMPRESS:
 		case DECOMPRESS:
 			if(OptimizerUtils.ALLOW_SCRIPT_LEVEL_COMPRESS_COMMAND){

src/main/java/org/apache/sysds/parser/DMLProgram.java

@@ -36,7 +36,13 @@ public class DMLProgram
 	private ArrayList<StatementBlock> _blocks;
 	private Map<String, FunctionDictionary<FunctionStatementBlock>> _namespaces;
 	private boolean _containsRemoteParfor;
-
+
+	/**
+	 * Session-wide differential privacy budget resolved at compile time during HOP construction.
+	 */
+	private Double _dpBudgetEpsilon;
+	private Double _dpBudgetDelta;
+
 	public DMLProgram(){
 		_blocks = new ArrayList<>();
 		_namespaces = new HashMap<>();
@@ -67,6 +73,23 @@ public void setContainsRemoteParfor(boolean flag) {
 	public boolean containsRemoteParfor() {
 		return _containsRemoteParfor;
 	}
+
+	public void setDPBudget(double epsilon, double delta) {
+		_dpBudgetEpsilon = epsilon;
+		_dpBudgetDelta = delta;
+	}
+
+	public boolean hasDPBudget() {
+		return _dpBudgetEpsilon != null;
+	}
+
+	public double getDPBudgetEpsilon() {
+		return _dpBudgetEpsilon;
+	}
+
+	public double getDPBudgetDelta() {
+		return _dpBudgetDelta;
+	}
 
 	public static boolean isInternalNamespace(String namespace) {
 		return DEFAULT_NAMESPACE.equals(namespace)

src/main/java/org/apache/sysds/parser/DMLTranslator.java

@@ -2015,6 +2015,8 @@ private Hop processParameterizedBuiltinFunctionExpression(ParameterizedBuiltinFu
 			case TRANSFORMMETA:
 			case PARAMSERV:
 			case AUTODIFF:
+			case DP_GAUSSIAN:
+			case DP_LAPLACE:
 				currBuiltinOp = new ParameterizedBuiltinOp(target.getName(), target.getDataType(),
 					target.getValueType(), ParamBuiltinOp.valueOf(source.getOpCode().name()), paramHops);
 				break;
@@ -2590,6 +2592,19 @@ else if ( sop.equalsIgnoreCase(Opcodes.NOTEQUAL.toString()) )
 		case DECOMPRESS:
 			currBuiltinOp = new UnaryOp(target.getName(), target.getDataType(), ValueType.FP64, OpOp1.DECOMPRESS, expr);
 			break;
+		case DP_SET_BUDGET:
+			// Resolved entirely at compile time; the budget is applied directly to the DMLProgram (reachable later from
+			// ExecutionContext via Program.getDMLProg())
+			if (_dmlProg.hasDPBudget())
+				throw new LanguageException(source.getOpCode() + ": dp_set_budget may only be called once per "
+					+ "script (already set to epsilon=" + _dmlProg.getDPBudgetEpsilon()
+					+ ", delta=" + _dmlProg.getDPBudgetDelta() + ")");
+			if (!(expr instanceof LiteralOp) || !(expr2 instanceof LiteralOp))
+				throw new LanguageException(source.getOpCode()
+					+ ": epsilon and delta must be compile-time numeric literals");
+			_dmlProg.setDPBudget(((LiteralOp) expr).getDoubleValue(), ((LiteralOp) expr2).getDoubleValue());
+			currBuiltinOp = expr; // echo epsilon back as confirmation
+			break;
 		case QUANTIZE_COMPRESS:
 			currBuiltinOp = new BinaryOp(target.getName(), target.getDataType(), target.getValueType(), OpOp2.valueOf(source.getOpCode().name()), expr, expr2);
 			break;

src/main/java/org/apache/sysds/parser/ParameterizedBuiltinFunctionExpression.java

@@ -268,6 +268,11 @@ public void validateExpression(HashMap<String, DataIdentifier> ids, HashMap<Stri
 			validateUnique(output, conditional);
 			break;
 
+		case DP_GAUSSIAN:
+		case DP_LAPLACE:
+			validateDpMechanism(output, conditional);
+			break;
+
 		default: //always unconditional (because unsupported operation)
 			//handle common issue of transformencode
 			if( getOpCode()==Builtins.TRANSFORMENCODE )
@@ -770,6 +775,54 @@ private void validateReplace(DataIdentifier output, boolean conditional) {
 		output.setDimensions(target.getOutput().getDim1(), target.getOutput().getDim2());
 	}
 
+	private void validateDpMechanism(DataIdentifier output, boolean conditional) {
+		// support X as the sole positional (unnamed) parameter, like lowerTri(X, ...)
+		HashMap<String, Expression> varParams = getVarParams();
+		if(varParams.containsKey(null))
+			varParams.put("target", varParams.remove(null));
+
+		boolean gaussian = getOpCode() == Builtins.DP_GAUSSIAN;
+		Set<String> valid = gaussian ?
+			CollectionUtils.asSet("target", "query", "sensitivity", "epsilon", "delta") :
+			CollectionUtils.asSet("target", "query", "sensitivity", "epsilon");
+		checkInvalidParameters(getOpCode(), varParams, valid);
+
+		Expression target = getVarParam("target");
+		checkTargetParam(target, conditional);
+		for(String param : valid)
+			if(!param.equals("target"))
+				checkScalarParam(getOpCode().getName(), param, conditional);
+
+		long[] dims = getDPOutputDims(getVarParam("query"), target.getOutput().getDim1(), target.getOutput().getDim2());
+		output.setDataType(DataType.MATRIX);
+		output.setValueType(ValueType.FP64);
+		output.setDimensions(dims[0], dims[1]);
+	}
+
+	/** Output dimensions of T %*% X for the given named query, X being n x d. */
+	private long[] getDPOutputDims(Expression queryExpr, long n, long d) {
+		// dp_gaussian/dp_laplace require the "query" parameter to be a compile-time string literal so that the output
+		// shape
+		// (and thus the transformation matrix T built at runtime) is known during validation.
+		if(!(queryExpr instanceof StringIdentifier))
+			raiseValidateError(getOpCode() + ": 'query' must be a string literal", false,
+				LanguageErrorCodes.INVALID_PARAMETERS);
+		String query = ((StringIdentifier) queryExpr).getValue();
+
+		switch(query) {
+			case "colMeans":
+			case "colSums":
+				return new long[] {1, d};
+			case "identity":
+				return new long[] {n, d};
+			default:
+				raiseValidateError(
+					getOpCode() + ": unknown query type '" + query + "' (expected colMeans, colSums, or identity)",
+					false, LanguageErrorCodes.INVALID_PARAMETERS);
+				return null; // unreachable
+		}
+	}
+
 	private void checkScalarParam(String group, String param, boolean conditional) {
 		Expression eparam = getVarParam(param);
 		if( eparam==null ) {