Skip to content

Add concurrency to doclet #139

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
lqiu96 wants to merge 15 commits into from
Closed

Add concurrency to doclet #139

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
88da391
feat: Increase initial map capacity
lqiu96 Jun 12, 2022
d1a4846
feat: Add thread safe element access
lqiu96 Jun 12, 2022
7cdc0a0
feat: Use common map for caching results
lqiu96 Jun 13, 2022
2baa68b
feat: Build overview.yml file in order
lqiu96 Jun 13, 2022
0f919f4
feat: make DocletRunner runnable for optios file and argfile
suztomo Jun 13, 2022
6bc8348
test: example file to contain single quotes
suztomo Jun 13, 2022
78b892a
feat: Revert previous changes
lqiu96 Jun 13, 2022
39d52fb
Merge remote-tracking branch 'suztomo/fix_param_parsing' into concurrent
lqiu96 Jun 13, 2022
beb01d0
feat: Move initial map capacity to a constant
lqiu96 Jun 13, 2022
e73ee71
feat: Remove debugging code
lqiu96 Jun 14, 2022
31db2cf
fix: Use new regex to capture java class name
lqiu96 Jul 12, 2022
91fbf09
Merge branch 'fix_regex' into concurrent
lqiu96 Jul 12, 2022
34df9e5
Merge branch 'main' of https://github.com/lqiu96/java-docfx-doclet in…
lqiu96 Jul 20, 2022
0630c6a
feat: Add configs for ExecutorService
lqiu96 Jul 20, 2022
ed326d6
Merge branch 'main' into concurrent
lqiu96 Jul 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion third_party/docfx-doclet-143274/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@

<groupId>com.microsoft</groupId>
<artifactId>docfx-doclet</artifactId>
<version>1.0-SNAPSHOT</version>
<version>1.7.0</version>

<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
Expand Down
22 changes: 10 additions & 12 deletions third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ClassBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,27 +15,25 @@
*/
package com.microsoft.build;

import static com.microsoft.build.BuilderUtil.LANGS;
import static com.microsoft.build.BuilderUtil.populateItemFields;

import com.microsoft.lookup.ClassItemsLookup;
import com.microsoft.lookup.ClassLookup;
import com.microsoft.model.MetadataFile;
import com.microsoft.model.MetadataFileItem;
import com.microsoft.model.Status;
import com.microsoft.model.TocItem;
import com.microsoft.model.TocTypeMap;
import com.microsoft.util.ElementUtil;
import com.microsoft.util.Utils;

import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.lang.model.element.Element;
import javax.lang.model.element.ElementKind;
import javax.lang.model.element.ExecutableElement;
import javax.lang.model.element.TypeElement;
import javax.lang.model.util.ElementFilter;
import java.util.List;
import java.util.function.Function;
import java.util.stream.Collectors;

import static com.microsoft.build.BuilderUtil.LANGS;
import static com.microsoft.build.BuilderUtil.populateItemFields;

class ClassBuilder {
private ElementUtil elementUtil;
Expand Down Expand Up @@ -102,7 +100,7 @@ private void addClassInfo(TypeElement classElement, MetadataFile classMetadataFi
}

void addConstructorsInfo(TypeElement classElement, MetadataFile classMetadataFile) {
for (ExecutableElement constructorElement : ElementFilter.constructorsIn(classElement.getEnclosedElements())) {
for (ExecutableElement constructorElement : ElementFilter.constructorsIn(elementUtil.getEnclosedElements(classElement))) {
MetadataFileItem constructorItem = buildMetadataFileItem(constructorElement);
constructorItem.setOverload(classItemsLookup.extractOverload(constructorElement));
constructorItem.setContent(classItemsLookup.extractConstructorContent(constructorElement));
Expand All @@ -115,7 +113,7 @@ void addConstructorsInfo(TypeElement classElement, MetadataFile classMetadataFil
}

private void addMethodsInfo(TypeElement classElement, MetadataFile classMetadataFile) {
ElementFilter.methodsIn(classElement.getEnclosedElements()).stream()
ElementFilter.methodsIn(elementUtil.getEnclosedElements(classElement)).stream()
.filter(methodElement -> !Utils.isPrivateOrPackagePrivate(methodElement))
.forEach(methodElement -> {
MetadataFileItem methodItem = buildMetadataFileItem(methodElement);
Expand All @@ -135,7 +133,7 @@ private void addMethodsInfo(TypeElement classElement, MetadataFile classMetadata
}

private void addFieldsInfo(TypeElement classElement, MetadataFile classMetadataFile) {
ElementFilter.fieldsIn(classElement.getEnclosedElements()).stream()
ElementFilter.fieldsIn(elementUtil.getEnclosedElements(classElement)).stream()
.filter(fieldElement -> !Utils.isPrivateOrPackagePrivate(fieldElement))
.forEach(fieldElement -> {
MetadataFileItem fieldItem = buildMetadataFileItem(fieldElement);
Expand Down Expand Up @@ -177,7 +175,7 @@ private void collect(TypeElement classElement, List<String> children,
Function<Iterable<? extends Element>, List<? extends Element>> selectFunc,
Function<? super Element, String> mapFunc) {

List<? extends Element> elements = selectFunc.apply(classElement.getEnclosedElements());
List<? extends Element> elements = selectFunc.apply(elementUtil.getEnclosedElements(classElement));
children.addAll(filterPrivateElements(elements).stream()
.map(mapFunc).collect(Collectors.toList()));
}
Expand Down
7 changes: 5 additions & 2 deletions third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/Lookup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,17 @@

public class Lookup {

private Map<String, String> globalLookup = new HashMap<>();
private Map<String, Map<String, String>> localLookupByFileName = new HashMap<>();
private static final int INITIAL_CAPACITY = 10000;
private final Map<String, String> globalLookup;
private final Map<String, Map<String, String>> localLookupByFileName;

private final String UID_PACKAGE_NAME_REGEXP = "^.*?\\.(?=[A-Z].*)";
private final String PARAM_PACKAGE_NAME_REGEXP = "(?<=[\\( ]).*?(?=[A-Z].*)";
private final String METHOD_PARAMS_REGEXP = "\\s[^\\s]+?(?=[,)])";

public Lookup(List<MetadataFile> packageMetadataFiles, List<MetadataFile> classMetadataFiles) {
this.globalLookup = new HashMap<>(INITIAL_CAPACITY);
this.localLookupByFileName = new HashMap<>(INITIAL_CAPACITY);
Comment on lines +24 to +25
Copy link
Copy Markdown
Member Author

@lqiu96 lqiu96 Jul 20, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Set a large initial capacity to try and reduce the number of copying done (capacity expansion)

consume(packageMetadataFiles);
consume(classMetadataFiles);
}
Expand Down
10 changes: 4 additions & 6 deletions third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/ReferenceBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ String getJavaReferenceHref(String uid) {

void updateExternalReferences(List<MetadataFile> classMetadataFiles) {
classMetadataFiles.forEach(file -> file.getReferences()
.forEach(ref -> updateExternalReference(ref)));
.forEach(this::updateExternalReference));
}

private void updateExternalReference(MetadataFileItem reference) {
Expand Down Expand Up @@ -245,15 +245,13 @@ void addOverloadReferences(MetadataFileItem item, MetadataFile classMetadataFile
*/
void expandComplexGenericsInReferences(MetadataFile classMetadataFile) {
Set<MetadataFileItem> additionalItems = new LinkedHashSet<>();
Iterator<MetadataFileItem> iterator = classMetadataFile.getReferences().iterator();
while (iterator.hasNext()) {
MetadataFileItem item = iterator.next();
for (MetadataFileItem item : classMetadataFile.getReferences()) {
String uid = item.getUid();
if (!uid.endsWith("*") && uid.contains("<")) {
List<String> classNames = splitUidWithGenericsIntoClassNames(uid);
additionalItems.addAll(classNames.stream()
.map(s -> new MetadataFileItem(s, classLookup.makeTypeShort(s), false))
.collect(Collectors.toSet()));
.map(s -> new MetadataFileItem(s, classLookup.makeTypeShort(s), false))
.collect(Collectors.toSet()));
}
}
// Remove items which already exist in 'items' section (compared by 'uid' field)
Expand Down
69 changes: 53 additions & 16 deletions third_party/docfx-doclet-143274/src/main/java/com/microsoft/build/YmlFilesBuilder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,20 @@
import com.microsoft.model.TocTypeMap;
import com.microsoft.util.ElementUtil;
import com.microsoft.util.FileUtil;
import java.util.Collections;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import javax.tools.Diagnostic.Kind;
import jdk.javadoc.doclet.DocletEnvironment;

import javax.lang.model.element.PackageElement;
import java.util.ArrayList;
import java.util.List;
import java.util.stream.Collectors;
import jdk.javadoc.doclet.Reporter;

import static com.microsoft.build.BuilderUtil.populateUidValues;

Expand All @@ -30,23 +38,30 @@ public class YmlFilesBuilder {
private PackageBuilder packageBuilder;
private ClassBuilder classBuilder;
private ReferenceBuilder referenceBuilder;
private Reporter reporter;
private ExecutorService executorService;
private int numThreads;

public YmlFilesBuilder(DocletEnvironment environment, String outputPath,
String[] excludePackages, String[] excludeClasses, String projectName, boolean disableChangelog) {
String[] excludePackages, String[] excludeClasses, String projectName,
boolean disableChangelog, int numThreads, Reporter reporter) {
this.environment = environment;
this.outputPath = outputPath;
this.elementUtil = new ElementUtil(excludePackages, excludeClasses);
this.packageLookup = new PackageLookup(environment);
this.projectName = projectName;
this.disableChangelog = disableChangelog;
this.projectBuilder = new ProjectBuilder(projectName);
ClassLookup classLookup = new ClassLookup(environment);
ClassLookup classLookup = new ClassLookup(environment, elementUtil);
this.referenceBuilder = new ReferenceBuilder(environment, classLookup, elementUtil);
this.packageBuilder = new PackageBuilder(packageLookup, outputPath, referenceBuilder);
this.classBuilder = new ClassBuilder(elementUtil, classLookup, new ClassItemsLookup(environment), outputPath, referenceBuilder);
this.reporter = reporter;
this.numThreads = numThreads;
this.classBuilder = new ClassBuilder(elementUtil, classLookup, new ClassItemsLookup(environment, elementUtil), outputPath, referenceBuilder);
}

public boolean build() {
executorService = Executors.newFixedThreadPool(numThreads);
// table of contents
TocFile tocFile = new TocFile(outputPath, projectName, disableChangelog);
// overview page
Expand All @@ -56,23 +71,45 @@ public boolean build() {
// packages
List<MetadataFileItem> packageItems = new ArrayList<>();
// class/enum/interface/etc. pages
List<MetadataFile> classMetadataFiles = new ArrayList<>();
List<MetadataFile> classMetadataFiles = Collections.synchronizedList(new ArrayList<>());

List<Future<?>> futureList = new ArrayList<>();
for (PackageElement packageElement :
elementUtil.extractPackageElements(environment.getIncludedElements())) {
String packageUid = packageLookup.extractUid(packageElement);
String packageStatus = packageLookup.extractStatus(packageElement);
TocItem packageTocItem = new TocItem(packageUid, packageUid, packageStatus);
// build package summary
packageMetadataFiles.add(packageBuilder.buildPackageMetadataFile(packageElement));
// add package summary to toc
packageTocItem.getItems().add(new TocItem(packageUid, "Package summary"));
tocFile.addTocItem(packageTocItem);
Future<?> future = executorService.submit(() -> {
String packageUid = packageLookup.extractUid(packageElement);
String packageStatus = packageLookup.extractStatus(packageElement);
TocItem packageTocItem = new TocItem(packageUid, packageUid, packageStatus);
// build package summary
packageMetadataFiles.add(packageBuilder.buildPackageMetadataFile(packageElement));
// add package summary to toc
packageTocItem.getItems().add(new TocItem(packageUid, "Package summary"));
tocFile.addTocItem(packageTocItem);

// build classes/interfaces/enums/exceptions/annotations
TocTypeMap typeMap = new TocTypeMap();
classBuilder.buildFilesForInnerClasses(packageElement, typeMap, classMetadataFiles);
packageTocItem.getItems().addAll(joinTocTypeItems(typeMap));
// build classes/interfaces/enums/exceptions/annotations
TocTypeMap typeMap = new TocTypeMap();
classBuilder.buildFilesForInnerClasses(packageElement, typeMap, classMetadataFiles);
packageTocItem.getItems().addAll(joinTocTypeItems(typeMap));
reporter.print(Kind.NOTE, "Finished running for " + packageElement + " on " + Thread.currentThread().getName());
});
futureList.add(future);
}

for (Future<?> future : futureList) {
try {
future.get();
} catch (InterruptedException e) {
e.printStackTrace();
} catch (ExecutionException e) {
e.printStackTrace();
}
}

executorService.shutdown();
try {
executorService.awaitTermination(Long.MAX_VALUE, TimeUnit.MICROSECONDS);
} catch (InterruptedException e) {
e.printStackTrace();
}

for (MetadataFile packageFile : packageMetadataFiles) {
Expand Down
12 changes: 11 additions & 1 deletion third_party/docfx-doclet-143274/src/main/java/com/microsoft/doclet/DocFxDoclet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,8 +27,9 @@ public boolean run(DocletEnvironment environment) {
reporter.print(Kind.NOTE, "Excluded classes: " + Arrays.toString(excludeClasses));
reporter.print(Kind.NOTE, "Project name: " + projectName);
reporter.print(Kind.NOTE, "Disable changelog: " + disableChangelog);
reporter.print(Kind.NOTE, "Num Threads: " + numThreads);

return (new YmlFilesBuilder(environment, outputPath, excludePackages, excludeClasses, projectName, disableChangelog)).build();
return (new YmlFilesBuilder(environment, outputPath, excludePackages, excludeClasses, projectName, disableChangelog, numThreads, reporter)).build();
}

@Override
Expand All @@ -41,6 +42,7 @@ public String getName() {
private String[] excludeClasses = {};
private String projectName;
private boolean disableChangelog;
private int numThreads = 10;

@Override
public Set<? extends Option> getSupportedOptions() {
Expand Down Expand Up @@ -89,6 +91,14 @@ public boolean process(String option, List<String> arguments) {
return true;
}
},
new CustomOption(
"Number Threads", Arrays.asList("-numThreads", "--num-threads"), "numThreads") {
@Override
public boolean process(String option, List<String> arguments) {
numThreads = Integer.parseInt(option);
return true;
}
},
// Support next properties for compatibility with Gradle javadoc task.
// According to javadoc spec - these properties used by StandardDoclet and used only when
// 'doclet' parameter not populated. But Gradle javadoc not align with this rule and
Expand Down
13 changes: 6 additions & 7 deletions third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/BaseLookup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import com.sun.source.doctree.LinkTree;
import com.sun.source.doctree.LiteralTree;
import com.sun.source.doctree.SeeTree;
import java.util.concurrent.ConcurrentHashMap;
import jdk.javadoc.doclet.DocletEnvironment;
import org.apache.commons.lang3.RegExUtils;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -32,6 +33,7 @@

public abstract class BaseLookup<T extends Element> {

private static final int INITIAL_CAPACITY = 500000;
protected final Map<ElementKind, String> elementKindLookup = new HashMap<>() {{
put(ElementKind.PACKAGE, "Namespace");
put(ElementKind.CLASS, "Class");
Expand All @@ -44,20 +46,17 @@ public abstract class BaseLookup<T extends Element> {
put(ElementKind.FIELD, "Field");
}};

protected Map<T, ExtendedMetadataFileItem> map = new HashMap<>();
protected Map<T, ExtendedMetadataFileItem> map;
protected final DocletEnvironment environment;

protected BaseLookup(DocletEnvironment environment) {
this.environment = environment;
this.map = new ConcurrentHashMap<>(INITIAL_CAPACITY);
}

protected ExtendedMetadataFileItem resolve(T key) {
ExtendedMetadataFileItem value = map.get(key);
if (value == null) {
value = buildMetadataFileItem(key);
map.put(key, value);
}
return value;
map.computeIfAbsent(key, this::buildMetadataFileItem);
return map.get(key);
}

protected abstract ExtendedMetadataFileItem buildMetadataFileItem(T key);
Expand Down
13 changes: 7 additions & 6 deletions third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassItemsLookup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import com.microsoft.model.MethodParameter;
import com.microsoft.model.Return;
import com.microsoft.util.CommentHelper;
import com.microsoft.util.ElementUtil;
import com.microsoft.util.Utils;
import com.sun.source.doctree.DocCommentTree;
import com.sun.source.doctree.DocTree;
Expand All @@ -29,13 +30,13 @@
public class ClassItemsLookup extends BaseLookup<Element> {
private Utils utils;

public ClassItemsLookup(DocletEnvironment environment) {
public ClassItemsLookup(DocletEnvironment environment, ElementUtil elementUtil) {
super(environment);
utils = new Utils(environment);
utils = new Utils(environment, elementUtil);
}

@Override
protected ExtendedMetadataFileItem buildMetadataFileItem(Element element) {
protected synchronized ExtendedMetadataFileItem buildMetadataFileItem(Element element) {
String packageName = determinePackageName(element);
TypeElement classElement = (TypeElement) element.getEnclosingElement();
String classQName = String.valueOf(classElement.getQualifiedName());
Expand All @@ -59,14 +60,14 @@ protected ExtendedMetadataFileItem buildMetadataFileItem(Element element) {
ExecutableElement exeElement = (ExecutableElement) element;
List<MethodParameter> parameters = extractParameters(exeElement);
String paramsString = parameters.stream()
.map(parameter -> String.format("%s %s", makeTypeShort(parameter.getType()), parameter.getId()))
.collect(Collectors.joining(", "));
.map(parameter -> String.format("%s %s", makeTypeShort(parameter.getType()), parameter.getId()))
.collect(Collectors.joining(", "));
String nameWithoutBrackets = elementQName.replaceAll("\\(.*\\)", "");
String methodName = String.format("%s(%s)", nameWithoutBrackets, paramsString);

result.setName(methodName);
result.setMethodContent(String.format("%s %s %s", modifiers,
makeTypeShort(String.valueOf(exeElement.getReturnType())), result.getName()));
makeTypeShort(String.valueOf(exeElement.getReturnType())), result.getName()));
result.setConstructorContent(String.format("%s %s", modifiers, result.getName()));
result.setParameters(parameters);
result.setExceptions(extractExceptions(exeElement));
Expand Down
11 changes: 8 additions & 3 deletions third_party/docfx-doclet-143274/src/main/java/com/microsoft/lookup/ClassLookup.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,9 @@
import com.microsoft.lookup.model.ExtendedMetadataFileItem;
import com.microsoft.model.MetadataFileItem;
import com.microsoft.model.TypeParameter;
import com.microsoft.util.ElementUtil;
import com.microsoft.util.Utils;
import java.util.Collections;
import jdk.javadoc.doclet.DocletEnvironment;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
Expand All @@ -25,12 +27,15 @@ public class ClassLookup extends BaseLookup<TypeElement> {

private static final String JAVA_LANG_OBJECT = "java.lang.Object";

public ClassLookup(DocletEnvironment environment) {
private final ElementUtil elementUtil;

public ClassLookup(DocletEnvironment environment, ElementUtil elementUtil) {
super(environment);
this.elementUtil = elementUtil;
}

@Override
protected ExtendedMetadataFileItem buildMetadataFileItem(TypeElement classElement) {
protected synchronized ExtendedMetadataFileItem buildMetadataFileItem(TypeElement classElement) {
List<ExtendedMetadataFileItem> inheritedMethods = new ArrayList<>();

String packageName = determinePackageName(classElement);
Expand Down Expand Up @@ -146,7 +151,7 @@ List<TypeParameter> determineTypeParameters(TypeElement element) {
}

void appendInheritedMethods(TypeElement element, List<ExtendedMetadataFileItem> inheritedMethods) {
List<? extends Element> members = element.getEnclosedElements();
List<? extends Element> members = elementUtil.getEnclosedElements(element);
Integer level = Optional.ofNullable(getMaxNestedLevel(inheritedMethods))
.orElse(0);

Expand Down
Loading