Skip to content

JS: Remove TypeScript parser memory limits #22991

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
paldepind wants to merge 1 commit into from
+3 −77
Closed

JS: Remove TypeScript parser memory limits #22991

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 0 additions & 14 deletions javascript/extractor/src/com/semmle/js/extractor/ExtractorConfig.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -224,9 +224,6 @@ public Set<String> getPredefinedGlobals() {
/** Should textual information be extracted into the lines/4 relation? */
private boolean extractLines;

/** Override amount of RAM to allocate to the TypeScript compiler. */
private int typescriptRam;

/** The default character encoding to use for parsing source files. */
private String defaultEncoding;

Expand Down Expand Up @@ -267,7 +264,6 @@ public ExtractorConfig(ExtractorConfig that) {
this.sourceType = that.sourceType;
this.htmlHandling = that.htmlHandling;
this.extractLines = that.extractLines;
this.typescriptRam = that.typescriptRam;
this.defaultEncoding = that.defaultEncoding;
this.virtualSourceRoot = that.virtualSourceRoot;
}
Expand Down Expand Up @@ -426,16 +422,6 @@ public ExtractorConfig withExtractLines(boolean extractLines) {
return res;
}

public int getTypeScriptRam() {
return typescriptRam;
}

public ExtractorConfig withTypeScriptRam(int ram) {
ExtractorConfig res = new ExtractorConfig(this);
res.typescriptRam = ram;
return res;
}

public String getDefaultEncoding() {
return defaultEncoding;
}
Expand Down
15 changes: 3 additions & 12 deletions javascript/extractor/src/com/semmle/js/extractor/Main.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,6 @@
import com.semmle.js.parser.ParsedProject;
import com.semmle.ts.extractor.TypeScriptParser;
import com.semmle.util.data.StringUtil;
import com.semmle.util.data.UnitParser;
import com.semmle.util.exception.ResourceError;
import com.semmle.util.exception.UserError;
import com.semmle.util.extraction.ExtractorOutputConfig;
Expand Down Expand Up @@ -60,9 +59,9 @@ public class Main {
private static final String P_SOURCE_TYPE = "--source-type";
private static final String P_TYPESCRIPT = "--typescript";
private static final String P_TYPESCRIPT_FULL = "--typescript-full";
private static final String P_TYPESCRIPT_RAM = "--typescript-ram";

// symbolic constants for deprecated command line parameter names
private static final String P_TYPESCRIPT_RAM = "--typescript-ram";
private static final String P_EXCLUDE_PATH = "--exclude-path";
private static final String P_TOLERATE_PARSE_ERRORS = "--tolerate-parse-errors";
private static final String P_NODEJS = "--nodejs";
Expand Down Expand Up @@ -130,7 +129,6 @@ public void run(String[] args) {
}

TypeScriptParser tsParser = extractorState.getTypeScriptParser();
tsParser.setTypescriptRam(extractorConfig.getTypeScriptRam());
if (containsTypeScriptFiles()) {
tsParser.verifyInstallation(!ap.has(P_QUIET));
}
Expand Down Expand Up @@ -373,10 +371,7 @@ private ArgsParser addArgs(ArgsParser argsParser) {
argsParser.addFlag(P_TYPESCRIPT, 0, "Enable basic TypesScript support.");
argsParser.addFlag(
P_TYPESCRIPT_FULL, 0, "Enable full TypeScript support with static type information.");
argsParser.addFlag(
P_TYPESCRIPT_RAM,
1,
"Amount of memory allocated to the TypeScript compiler process. The default is 1G.");
argsParser.addToleratedFlag(P_TYPESCRIPT_RAM, 1);
argsParser.addToleratedFlag(P_DEBUG_EXCLUSIONS, 0);
argsParser.addTrailingParam("files", "Files and directories to extract.");
return argsParser;
Expand Down Expand Up @@ -415,11 +410,7 @@ private ExtractorConfig parseJSOptions(ArgsParser ap) {
ap.has(P_EXPERIMENTAL) ? HtmlPopulator.Config.ALL : HtmlPopulator.Config.ELEMENTS))
.withFileType(getFileType(ap))
.withSourceType(ap.getEnum(P_SOURCE_TYPE, SourceType.class, SourceType.AUTO))
.withExtractLines(ap.has(P_EXTRACT_PROGRAM_TEXT))
.withTypeScriptRam(
ap.has(P_TYPESCRIPT_RAM)
? UnitParser.parseOpt(ap.getString(P_TYPESCRIPT_RAM), UnitParser.MEGABYTES)
: 0);
.withExtractLines(ap.has(P_EXTRACT_PROGRAM_TEXT));
if (ap.has(P_DEFAULT_ENCODING)) cfg = cfg.withDefaultEncoding(ap.getString(P_DEFAULT_ENCODING));

// Make a usable virtual source root mapping.
Expand Down
36 changes: 0 additions & 36 deletions javascript/extractor/src/com/semmle/ts/extractor/TypeScriptParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,6 @@
import com.semmle.js.parser.ParsedProject;
import com.semmle.js.parser.JSParser.Result;
import com.semmle.util.data.StringUtil;
import com.semmle.util.data.UnitParser;
import com.semmle.util.exception.CatastrophicError;
import com.semmle.util.exception.Exceptions;
import com.semmle.util.exception.InterruptedError;
Expand Down Expand Up @@ -96,24 +95,6 @@ public class TypeScriptParser {
*/
public static final String TYPESCRIPT_RETRIES_VAR = "SEMMLE_TYPESCRIPT_RETRIES";

/**
* An environment variable (without the <code>SEMMLE_</code> or <code>LGTM_</code> prefix), that can be
* set to indicate the maximum heap space usable by the Node.js process, in addition to its
* "reserve memory".
*
* <p>Defaults to 2000 MB (for a total heap space of 2400 MB by default).
*/
public static final String TYPESCRIPT_RAM_SUFFIX = "TYPESCRIPT_RAM";

/**
* An environment variable (without the <code>SEMMLE_</code> or <code>LGTM_</code> prefix), that can be
* set to indicate the additional heap space added to <code>TYPESCRIPT_RAM</code> when setting the
* Node.js heap limit.
*
* <p>Defaults to 400 MB (for a total heap space of 2400 MB by default).
*/
public static final String TYPESCRIPT_RAM_RESERVE_SUFFIX = "TYPESCRIPT_RAM_RESERVE";

/**
* An environment variable with additional VM arguments to pass to the Node process.
*
Expand Down Expand Up @@ -154,17 +135,9 @@ public class TypeScriptParser {
*/
private List<String> nodeJsRuntimeExtraArgs = Collections.emptyList();

/** If non-zero, we use this instead of relying on the corresponding environment variable. */
private int typescriptRam = 0;

/** Metadata requested immediately after starting the TypeScript parser. */
private TypeScriptParserMetadata metadata;

/** Sets the amount of RAM to allocate to the TypeScript compiler.s */
public void setTypescriptRam(int megabytes) {
this.typescriptRam = megabytes;
}

/**
* Verifies that Node.js and TypeScript are installed and throws an exception otherwise.
*
Expand Down Expand Up @@ -274,14 +247,6 @@ private List<String> getNodeJsRuntimeInvocation(String... args) {
private void setupParserWrapper() {
verifyNodeInstallation();

int mainMemoryMb =
typescriptRam != 0
? typescriptRam
: EnvironmentVariables.getMegabyteCountFromPrefixedEnv(TYPESCRIPT_RAM_SUFFIX, 2000);
int reserveMemoryMb = EnvironmentVariables.getMegabyteCountFromPrefixedEnv(TYPESCRIPT_RAM_RESERVE_SUFFIX, 400);

System.out.println("Memory for TypeScript process: " + mainMemoryMb + " MB, and " + reserveMemoryMb + " MB reserve");

File parserWrapper = getParserWrapper();

String debugFlagString = Env.systemEnv().getNonEmpty(TYPESCRIPT_NODE_FLAGS);
Expand All @@ -297,7 +262,6 @@ private void setupParserWrapper() {
}

List<String> cmd = getNodeJsRuntimeInvocation();
cmd.add("--max_old_space_size=" + (mainMemoryMb + reserveMemoryMb));
cmd.addAll(debugFlags);
cmd.add(parserWrapper.getAbsolutePath());

Expand Down
7 changes: 0 additions & 7 deletions javascript/resources/tools/autobuild.cmd
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,6 @@ SETLOCAL EnableDelayedExpansion

set jvm_args=-Xss16m

rem If CODEQL_RAM is set, use half for Java and half for TS.
if NOT [%CODEQL_RAM%] == [] (
set /a "half_ram=CODEQL_RAM/2"
set LGTM_TYPESCRIPT_RAM=!half_ram!
set jvm_args=!jvm_args! -Xmx!half_ram!m
)

rem If CODEQL_THREADS is set, propagate via LGTM_THREADS.
if NOT [%CODEQL_THREADS%] == [] (
set LGTM_THREADS=%CODEQL_THREADS%
Expand Down
8 changes: 0 additions & 8 deletions javascript/resources/tools/autobuild.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,6 @@ set -eu

jvm_args=-Xss16m

# If CODEQL_RAM is set, use half for Java and half for TS.
if [ -n "${CODEQL_RAM:-}" ] ; then
half_ram="$(( CODEQL_RAM / 2 ))"
LGTM_TYPESCRIPT_RAM="$half_ram"
export LGTM_TYPESCRIPT_RAM
jvm_args="$jvm_args -Xmx${half_ram}m"
fi

# If CODEQL_THREADS is set, propagate via LGTM_THREADS.
if [ -n "${CODEQL_THREADS:-}" ] ; then
LGTM_THREADS="$CODEQL_THREADS"
Expand Down
Loading