Skip to content

[ML] Convert encodeIds flag in TreeEncoder to configuration property #806

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
AliRana30 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

[ML] Convert encodeIds flag in TreeEncoder to configuration property #806

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MLContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ public void execute(final WayangPlan wayangPlan, final String... udfJars) {
final Optional<String> originalOption = config.getOptionalStringProperty("wayang.ml.experience.original");

final OneHotMappings mappings = new OneHotMappings();
final TreeEncoder encoder = new TreeEncoder(mappings);
final String original = originalOption.orElse(encoder.encode(wayangPlan, wayangJob.getOptimizationContext(), false).toString());
final TreeEncoder encoder = new TreeEncoder(mappings, config);
final String original = originalOption.orElse(encoder.encode(wayangPlan, wayangJob.getOptimizationContext()).toString());

final Optional<String> choicesOption = config
.getOptionalStringProperty("wayang.ml.experience.with-platforms");
Expand Down
8 changes: 8 additions & 0 deletions wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/MachineLearning.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package org.apache.wayang.ml;

import java.io.InputStream;
import java.util.Collection;
import java.util.Collections;

Expand All @@ -26,6 +27,7 @@
import org.apache.wayang.core.optimizer.channels.ChannelConversion;
import org.apache.wayang.core.platform.Platform;
import org.apache.wayang.core.plugin.Plugin;
import org.apache.wayang.core.util.ReflectionUtils;
import org.apache.wayang.java.platform.JavaPlatform;
import org.apache.wayang.spark.platform.SparkPlatform;

Expand All @@ -34,6 +36,8 @@
*/
public class MachineLearning {

private static final String DEFAULT_CONFIG_FILE = "wayang-ml-defaults.properties";

/**
* Enables use with the {@link JavaPlatform} and {@link SparkPlatform}.
*/
Expand All @@ -56,6 +60,10 @@ public Collection<ChannelConversion> getChannelConversions() {

@Override
public void setProperties(final Configuration configuration) {
final InputStream is = ReflectionUtils.loadResource(DEFAULT_CONFIG_FILE);
if (is != null) {
configuration.load(is);
}
}
};

Expand Down
41 changes: 38 additions & 3 deletions wayang-plugins/wayang-ml/src/main/java/org/apache/wayang/ml/encoding/TreeEncoder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import java.util.Map;
import java.util.Queue;

import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.exception.WayangException;
import org.apache.wayang.core.optimizer.OptimizationContext;
import org.apache.wayang.core.optimizer.enumeration.PlanImplementation;
Expand All @@ -40,27 +41,57 @@
import org.apache.wayang.core.platform.Junction;

public class TreeEncoder {

public static final String ENCODE_IDS_PROPERTY = "wayang.ml.encoding.encode-ids";

private final OneHotMappings mappings;

private Configuration configuration;

public TreeEncoder(final OneHotMappings mappings) {
this(mappings, null);
}

public TreeEncoder(final OneHotMappings mappings, final Configuration configuration) {
this.mappings = mappings;
this.configuration = configuration;
}

public OneHotMappings getMappings() {
return this.mappings;
}

public Configuration getConfiguration() {
return this.configuration;
}

public void setConfiguration(final Configuration configuration) {
this.configuration = configuration;
}

private boolean resolveEncodeIds(final OptimizationContext optimizationContext) {
final Configuration config = this.configuration != null
? this.configuration
: (optimizationContext != null ? optimizationContext.getConfiguration() : null);
if (config != null) {
return config.getOptionalBooleanProperty(ENCODE_IDS_PROPERTY)
.orElseGet(() -> config.getBooleanProperty("wayang.ml.encode-ids", false));
}
return false;
}

public TreeNode encode(final PlanImplementation plan) {
return this.encode(plan, this.resolveEncodeIds(plan.getOptimizationContext()));
}

public TreeNode encode(final PlanImplementation plan, final boolean encodeIds) {
final List<TreeNode> result = new ArrayList<TreeNode>();

final HashMap<Operator, Collection<Operator>> tree = new HashMap<>();
final List<ExecutionOperator> sinks = plan.getOperators().stream().filter(Operator::isSink).toList();

final Map<OutputSlot<?>, Junction> junctions = plan.getJunctions();

// TODO: convert to config
final boolean encodeIds = false;

for (final Operator sink : sinks) {
final TreeNode sinkNode = traversePIOperator(sink, plan.getOptimizationContext(), encodeIds, junctions,
tree);
Expand All @@ -77,6 +108,10 @@ public TreeNode encode(final PlanImplementation plan) {
return resultNode;
}

public TreeNode encode(final WayangPlan plan, final OptimizationContext optimizationContext) {
return this.encode(plan, optimizationContext, this.resolveEncodeIds(optimizationContext));
}

public TreeNode encode(final WayangPlan plan, final OptimizationContext optimizationContext,
final boolean encodeIds) {
final List<TreeNode> result = new ArrayList<TreeNode>();
Expand Down
1 change: 1 addition & 0 deletions wayang-plugins/wayang-ml/src/main/resources/wayang-ml-defaults.properties
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,4 +73,5 @@ wayang.ml.experience.enabled = false
wayang.ml.executions.file = /var/www/html/data/executions.txt
wayang.ml.optimizations.file = /var/www/html/data/optmizations.txt
wayang.ml.experience.file = /var/www/html/data/experience/experience-vae.txt
wayang.ml.encoding.encode-ids = false
org.apache.logging.log4j.level = INFO
97 changes: 97 additions & 0 deletions wayang-plugins/wayang-ml/src/test/java/org/apache/wayang/ml/test/TreeEncoderTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,15 @@

import java.io.IOException;
import java.net.URISyntaxException;
import java.util.Collection;
import java.util.LinkedList;
import java.util.List;

import org.apache.wayang.basic.data.Tuple2;
import org.apache.wayang.core.api.Configuration;
import org.apache.wayang.core.api.Job;
import org.apache.wayang.core.api.WayangContext;
import org.apache.wayang.core.optimizer.enumeration.PlanImplementation;
import org.apache.wayang.core.plan.executionplan.ExecutionPlan;
import org.apache.wayang.core.plan.wayangplan.WayangPlan;
import org.apache.wayang.java.Java;
Expand Down Expand Up @@ -57,5 +59,100 @@ public void testTreeEncoding() throws IOException, URISyntaxException {

Assertions.assertNotNull(exPlan);
Assertions.assertNotNull(encoded);
Assertions.assertEquals(0L, encoded.encoded[0]);
}

@Test
public void testEncodePlanImplementationWithDefaultConfig() throws IOException, URISyntaxException {
final List<Tuple2<String, Integer>> collector = new LinkedList<>();
final Configuration config = new Configuration();
final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString();
final WayangPlan wayangPlan = createWayangPlan(filePath, collector);
final WayangContext wayangContext = new WayangContext(config);
wayangContext.register(Java.basicPlugin());
wayangContext.register(Spark.basicPlugin());

final Collection<PlanImplementation> planImplementations = buildPlanImplementations(wayangPlan, wayangContext);
Assertions.assertFalse(planImplementations.isEmpty());

final TreeEncoder encoder = new TreeEncoder(new OneHotMappings());
for (final PlanImplementation planImplementation : planImplementations) {
final TreeNode encoded = encoder.encode(planImplementation);
Assertions.assertNotNull(encoded);
Assertions.assertEquals(0L, encoded.encoded[0]);
}
}

@Test
public void testEncodePlanImplementationWithEncodeIdsEnabled() throws IOException, URISyntaxException {
final List<Tuple2<String, Integer>> collector = new LinkedList<>();
final Configuration config = new Configuration();
config.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true");
final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString();
final WayangPlan wayangPlan = createWayangPlan(filePath, collector);
final WayangContext wayangContext = new WayangContext(config);
wayangContext.register(Java.basicPlugin());
wayangContext.register(Spark.basicPlugin());

final Collection<PlanImplementation> planImplementations = buildPlanImplementations(wayangPlan, wayangContext);
Assertions.assertFalse(planImplementations.isEmpty());

final TreeEncoder encoder = new TreeEncoder(new OneHotMappings());
for (final PlanImplementation planImplementation : planImplementations) {
final TreeNode encoded = encoder.encode(planImplementation);
Assertions.assertNotNull(encoded);
Assertions.assertNotEquals(0L, encoded.encoded[0]);
}
}

@Test
public void testEncodeWayangPlanWithConfiguration() throws IOException, URISyntaxException {
final List<Tuple2<String, Integer>> collector = new LinkedList<>();
final Configuration config = new Configuration();
config.setProperty("wayang.ml.encode-ids", "true");
final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString();
final WayangPlan wayangPlan = createWayangPlan(filePath, collector);
final WayangContext wayangContext = new WayangContext(config);
final Job wayangJob = wayangContext.createJob("", wayangPlan, "");
wayangContext.register(Java.basicPlugin());
wayangContext.register(Spark.basicPlugin());

wayangJob.buildInitialExecutionPlan();

final TreeEncoder encoder = new TreeEncoder(new OneHotMappings());
final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext());

Assertions.assertNotNull(encoded);
Assertions.assertNotEquals(0L, encoded.encoded[0]);
}

@Test
public void testTreeEncoderConstructorWithConfiguration() throws IOException, URISyntaxException {
final List<Tuple2<String, Integer>> collector = new LinkedList<>();
final Configuration config = new Configuration();
config.setProperty(TreeEncoder.ENCODE_IDS_PROPERTY, "true");
final String filePath = JavaExecutionMLTest.class.getResource("/README.md").toURI().toString();
final WayangPlan wayangPlan = createWayangPlan(filePath, collector);
final WayangContext wayangContext = new WayangContext(config);
final Job wayangJob = wayangContext.createJob("", wayangPlan, "");
wayangContext.register(Java.basicPlugin());
wayangContext.register(Spark.basicPlugin());

wayangJob.buildInitialExecutionPlan();

final TreeEncoder encoder = new TreeEncoder(new OneHotMappings(), config);
Assertions.assertEquals(config, encoder.getConfiguration());

final TreeNode encoded = encoder.encode(wayangPlan, wayangJob.getOptimizationContext());
Assertions.assertNotNull(encoded);
Assertions.assertNotEquals(0L, encoded.encoded[0]);
}

@Test
public void testMachineLearningPluginSetProperties() {
final Configuration config = new Configuration();
org.apache.wayang.ml.MachineLearning.plugin().setProperties(config);
Assertions.assertEquals(false, config.getBooleanProperty(TreeEncoder.ENCODE_IDS_PROPERTY));
Assertions.assertEquals(100L, config.getLongProperty("wayang.ml.tuple.average-size"));
}
}