Skip to content

Commit df988e4

Browse files
tamasvajktamasvajk
authored
Merge pull request #14351 from tamasvajk/csharp/standalone-compilation
C#: Extract compilation DB entity in standalone mode
2 parents c365f45 + b2514b3 commit df988e4

17 files changed

Lines changed: 151 additions & 101 deletions

File tree

‎csharp/extractor/Semmle.Extraction.CSharp.DependencyFetching/DependencyManager.cs‎

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ public DependencyManager(string srcDir, IDependencyOptions options, ILogger logg
6060
this.progressMonitor.FindingFiles(srcDir);
6161

6262
packageDirectory = new TemporaryDirectory(ComputeTempDirectory(sourceDir.FullName));
63-
tempWorkingDirectory = new TemporaryDirectory(GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
63+
tempWorkingDirectory = new TemporaryDirectory(FileUtils.GetTemporaryWorkingDirectory(out cleanupTempWorkingDirectory));
6464

6565
var allFiles = GetAllFiles();
6666
var binaryFileExtensions = new HashSet<string>(new[] { ".dll", ".exe" }); // TODO: add more binary file extensions.
@@ -286,22 +286,6 @@ private static string ComputeTempDirectory(string srcDir)
286286
return Path.Combine(Path.GetTempPath(), "GitHub", "packages", sb.ToString());
287287
}
288288

289-
private static string GetTemporaryWorkingDirectory(out bool cleanupTempWorkingDirectory)
290-
{
291-
cleanupTempWorkingDirectory = false;
292-
var tempFolder = EnvironmentVariables.GetScratchDirectory();
293-
294-
if (string.IsNullOrEmpty(tempFolder))
295-
{
296-
var tempPath = Path.GetTempPath();
297-
var name = Guid.NewGuid().ToString("N").ToUpper();
298-
tempFolder = Path.Combine(tempPath, "GitHub", name);
299-
cleanupTempWorkingDirectory = true;
300-
}
301-
302-
return tempFolder;
303-
}
304-
305289
/// <summary>
306290
/// Creates a temporary directory with the given subfolder name.
307291
/// The created directory might be inside the repo folder, and it is deleted when the object is disposed.

‎csharp/extractor/Semmle.Extraction.CSharp.Standalone/Extractor.cs‎

Lines changed: 37 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -30,29 +30,46 @@ private static void AnalyseStandalone(
3030
IProgressMonitor progressMonitor,
3131
Stopwatch stopwatch)
3232
{
33-
CSharp.Extractor.Analyse(stopwatch, analyser, options,
34-
references => GetResolvedReferencesStandalone(referencePaths, references),
35-
(analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
36-
(syntaxTrees, references) => CSharpCompilation.Create(
37-
"csharp.dll", syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true)
38-
),
39-
(compilation, options) => analyser.Initialize(compilation, options),
40-
() => { },
41-
_ => { },
42-
() =>
43-
{
44-
foreach (var type in analyser.MissingNamespaces)
45-
{
46-
progressMonitor.MissingNamespace(type);
47-
}
33+
var output = FileUtils.CreateTemporaryFile(".dll", out var shouldCleanUpContainingFolder);
4834

49-
foreach (var type in analyser.MissingTypes)
35+
try
36+
{
37+
CSharp.Extractor.Analyse(stopwatch, analyser, options,
38+
references => GetResolvedReferencesStandalone(referencePaths, references),
39+
(analyser, syntaxTrees) => CSharp.Extractor.ReadSyntaxTrees(sources, analyser, null, null, syntaxTrees),
40+
(syntaxTrees, references) => CSharpCompilation.Create(
41+
output.Name, syntaxTrees, references, new CSharpCompilationOptions(OutputKind.ConsoleApplication, allowUnsafe: true)
42+
),
43+
(compilation, options) => analyser.Initialize(output.FullName, compilation, options),
44+
_ => { },
45+
() =>
5046
{
51-
progressMonitor.MissingType(type);
47+
foreach (var type in analyser.MissingNamespaces)
48+
{
49+
progressMonitor.MissingNamespace(type);
50+
}
51+
52+
foreach (var type in analyser.MissingTypes)
53+
{
54+
progressMonitor.MissingType(type);
55+
}
56+
57+
progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
58+
});
59+
}
60+
finally
61+
{
62+
try
63+
{
64+
FileUtils.TryDelete(output.FullName);
65+
if (shouldCleanUpContainingFolder)
66+
{
67+
output.Directory?.Delete(true);
5268
}
53-
54-
progressMonitor.MissingSummary(analyser.MissingTypes.Count(), analyser.MissingNamespaces.Count());
55-
});
69+
}
70+
catch
71+
{ }
72+
}
5673
}
5774

5875
private static void ExtractStandalone(

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Assembly.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ private Assembly(Context cx, Microsoft.CodeAnalysis.Location? init)
1616
if (init is null)
1717
{
1818
// This is the output assembly
19-
assemblyPath = ((TracingExtractor)cx.Extractor).OutputPath;
19+
assemblyPath = cx.Extractor.OutputPath;
2020
assembly = cx.Compilation.Assembly;
2121
}
2222
else
@@ -63,8 +63,6 @@ private class AssemblyConstructorFactory : CachedEntityFactory<Microsoft.CodeAna
6363

6464
public static Assembly CreateOutputAssembly(Context cx)
6565
{
66-
if (cx.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
67-
throw new InternalError("Attempting to create the output assembly in standalone extraction mode");
6866
return AssemblyConstructorFactory.Instance.CreateEntity(cx, outputAssemblyCacheKey, null);
6967
}
7068

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Attribute.cs‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,7 @@ public override void Populate(TextWriter trapFile)
6363

6464
if (attributeSyntax is not null)
6565
{
66-
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
67-
{
68-
trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
69-
}
66+
trapFile.attribute_location(this, Assembly.CreateOutputAssembly(Context));
7067

7168
TypeMention.Create(Context, attributeSyntax.Name, this, type);
7269
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/CachedSymbol.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ public virtual IEnumerable<Extraction.Entities.Location> Locations
9797
{
9898
// Some built in operators lack locations, so loc is null.
9999
yield return Context.CreateLocation(ReportingLocation);
100-
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && loc.Kind == LocationKind.SourceFile)
100+
if (loc.Kind == LocationKind.SourceFile)
101101
yield return Assembly.CreateOutputAssembly(Context);
102102
}
103103
}

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/PreprocessorDirectives/PreprocessorDirective.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,8 @@ public sealed override void Populate(TextWriter trapFile)
1515
trapFile.preprocessor_directive_active(this, Symbol.IsActive);
1616
trapFile.preprocessor_directive_location(this, Context.CreateLocation(ReportingLocation));
1717

18-
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone))
19-
{
20-
var compilation = Compilation.Create(Context);
21-
trapFile.preprocessor_directive_compilation(this, compilation);
22-
}
18+
var compilation = Compilation.Create(Context);
19+
trapFile.preprocessor_directive_compilation(this, compilation);
2320
}
2421

2522
protected abstract void PopulatePreprocessor(TextWriter trapFile);

‎csharp/extractor/Semmle.Extraction.CSharp/Entities/Types/NamedType.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,7 @@ public override IEnumerable<Extraction.Entities.Location> Locations
108108
foreach (var l in GetLocations(Symbol))
109109
yield return Context.CreateLocation(l);
110110

111-
if (!Context.Extractor.Mode.HasFlag(ExtractorMode.Standalone) && Symbol.DeclaringSyntaxReferences.Any())
111+
if (Symbol.DeclaringSyntaxReferences.Any())
112112
yield return Assembly.CreateOutputAssembly(Context);
113113
}
114114
}

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Analyser.cs‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ public class Analyser : IDisposable
1919
protected Extraction.Extractor? extractor;
2020
protected CSharpCompilation? compilation;
2121
protected CommonOptions? options;
22+
private protected Entities.Compilation? compilationEntity;
23+
private IDisposable? compilationTrapFile;
2224

2325
private readonly object progressMutex = new object();
2426

@@ -226,8 +228,35 @@ private void DoExtractTree(SyntaxTree tree)
226228
}
227229
}
228230

231+
private void DoAnalyseCompilation()
232+
{
233+
try
234+
{
235+
var assemblyPath = extractor.OutputPath;
236+
var transformedAssemblyPath = PathTransformer.Transform(assemblyPath);
237+
var assembly = compilation.Assembly;
238+
var trapWriter = transformedAssemblyPath.CreateTrapWriter(Logger, options.TrapCompression, discardDuplicates: false);
239+
compilationTrapFile = trapWriter; // Dispose later
240+
var cx = new Context(extractor, compilation.Clone(), trapWriter, new AssemblyScope(assembly, assemblyPath), addAssemblyTrapPrefix);
241+
242+
compilationEntity = Entities.Compilation.Create(cx);
243+
}
244+
catch (Exception ex) // lgtm[cs/catch-of-all-exceptions]
245+
{
246+
Logger.Log(Severity.Error, " Unhandled exception analyzing {0}: {1}", "compilation", ex);
247+
}
248+
}
249+
229250
#nullable restore warnings
230251

252+
/// <summary>
253+
/// Extracts compilation-wide entities, such as compilations and compiler diagnostics.
254+
/// </summary>
255+
public void AnalyseCompilation()
256+
{
257+
extractionTasks.Add(() => DoAnalyseCompilation());
258+
}
259+
231260
private static bool FileIsUpToDate(string src, string dest)
232261
{
233262
return File.Exists(dest) &&
@@ -275,6 +304,8 @@ public virtual void Dispose()
275304
Logger.Log(Severity.Info, "EXTRACTION SUCCEEDED in {0}", stopWatch.Elapsed);
276305

277306
Logger.Dispose();
307+
308+
compilationTrapFile?.Dispose();
278309
}
279310

280311
/// <summary>

‎csharp/extractor/Semmle.Extraction.CSharp/Extractor/Extractor.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -302,7 +302,6 @@ public static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOpt
302302
Func<Analyser, List<SyntaxTree>, IEnumerable<Action>> getSyntaxTreeTasks,
303303
Func<IEnumerable<SyntaxTree>, IEnumerable<MetadataReference>, CSharpCompilation> getCompilation,
304304
Action<CSharpCompilation, CommonOptions> initializeAnalyser,
305-
Action analyseCompilation,
306305
Action<Entities.PerformanceMetrics> logPerformance,
307306
Action postProcess)
308307
{
@@ -332,7 +331,7 @@ public static ExitCode Analyse(Stopwatch stopwatch, Analyser analyser, CommonOpt
332331
var compilation = getCompilation(syntaxTrees, references);
333332

334333
initializeAnalyser(compilation, options);
335-
analyseCompilation();
334+
analyser.AnalyseCompilation();
336335
analyser.AnalyseReferences();
337336

338337
foreach (var tree in compilation.SyntaxTrees)
@@ -416,7 +415,6 @@ private static ExitCode AnalyseTracing(
416415
);
417416
},
418417
(compilation, options) => analyser.EndInitialize(compilerArguments, options, compilation),
419-
() => analyser.AnalyseCompilation(),
420418
performance => analyser.LogPerformance(performance),
421419
() => { });
422420
}

csharp/extractor/Semmle.Extraction.CSharp.Standalone/StandaloneAnalyser.cs renamed to csharp/extractor/Semmle.Extraction.CSharp/Extractor/StandaloneAnalyser.cs

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1+
using System;
12
using System.Collections.Generic;
3+
using System.IO;
24
using Microsoft.CodeAnalysis.CSharp;
35
using Semmle.Util.Logging;
46

@@ -11,13 +13,15 @@ public StandaloneAnalyser(IProgressMonitor pm, ILogger logger, bool addAssemblyT
1113
{
1214
}
1315

14-
public void Initialize(CSharpCompilation compilationIn, CommonOptions options)
16+
public void Initialize(string outputPath, CSharpCompilation compilationIn, CommonOptions options)
1517
{
1618
compilation = compilationIn;
17-
extractor = new StandaloneExtractor(Logger, PathTransformer, options);
19+
extractor = new StandaloneExtractor(outputPath, Logger, PathTransformer, options);
1820
this.options = options;
1921
LogExtractorInfo(Extraction.Extractor.Version);
2022
SetReferencePaths();
23+
24+
Entities.Compilation.Settings = (Directory.GetCurrentDirectory(), Array.Empty<string>());
2125
}
2226

2327
#nullable disable warnings

0 commit comments

Comments
 (0)