-
Notifications
You must be signed in to change notification settings - Fork 73
Use Worker API for process isolation in Spotless
#1492
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: develop
Are you sure you want to change the base?
Changes from all commits
12708bb
ad55e12
712e282
727bd5e
b640d6c
267b460
07b55ab
803e5fb
f36a1fc
0390594
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| /* | ||
| * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.palantir.javaformat.gradle.spotless; | ||
|
|
||
| import org.gradle.api.file.ConfigurableFileCollection; | ||
| import org.gradle.api.file.RegularFileProperty; | ||
| import org.gradle.workers.WorkParameters; | ||
|
|
||
| public interface FormatJavaParameters extends WorkParameters { | ||
|
|
||
| RegularFileProperty getInputFile(); | ||
|
|
||
| RegularFileProperty getOutputFile(); | ||
|
|
||
| ConfigurableFileCollection getFormatterClasspath(); | ||
| } |
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,68 @@ | ||
| /* | ||
| * (c) Copyright 2025 Palantir Technologies Inc. All rights reserved. | ||
| * | ||
| * Licensed under the Apache License, Version 2.0 (the "License"); | ||
| * you may not use this file except in compliance with the License. | ||
| * You may obtain a copy of the License at | ||
| * | ||
| * http://www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * Unless required by applicable law or agreed to in writing, software | ||
| * distributed under the License is distributed on an "AS IS" BASIS, | ||
| * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| * See the License for the specific language governing permissions and | ||
| * limitations under the License. | ||
| */ | ||
|
|
||
| package com.palantir.javaformat.gradle.spotless; | ||
|
|
||
| import com.google.common.base.Suppliers; | ||
| import com.palantir.javaformat.java.FormatterService; | ||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.io.UncheckedIOException; | ||
| import java.nio.file.Files; | ||
| import java.util.ServiceLoader; | ||
| import java.util.function.Supplier; | ||
| import org.gradle.api.logging.Logger; | ||
| import org.gradle.api.logging.Logging; | ||
| import org.gradle.workers.WorkAction; | ||
|
|
||
| public abstract class FormatJavaWorkAction implements WorkAction<FormatJavaParameters> { | ||
|
|
||
| private static final Logger logger = Logging.getLogger(FormatJavaWorkAction.class); | ||
|
|
||
| private static final Supplier<FormatterService> formatterService = Suppliers.memoize(() -> { | ||
| logger.info("Loading FormatterService in worker daemon"); | ||
| return ServiceLoader.load(FormatterService.class, FormatJavaWorkAction.class.getClassLoader()) | ||
| .findFirst() | ||
| .orElseThrow(() -> new IllegalStateException( | ||
| "No FormatterService found. Ensure palantir-java-format is on the worker classpath.")); | ||
| }); | ||
|
|
||
| @Override | ||
| public void execute() { | ||
| FormatJavaParameters params = getParameters(); | ||
|
|
||
| try { | ||
| File inputFile = params.getInputFile().get().getAsFile(); | ||
| File outputFile = params.getOutputFile().get().getAsFile(); | ||
|
|
||
| String input = Files.readString(inputFile.toPath()); | ||
|
|
||
| logger.debug("Formatting file with worker process: {}", inputFile.getName()); | ||
|
|
||
| String output = formatterService.get().formatSourceReflowStringsAndFixImports(input); | ||
|
|
||
| Files.writeString(outputFile.toPath(), output); | ||
|
|
||
| } catch (IOException e) { | ||
| throw new UncheckedIOException("Failed to format Java file", e); | ||
| } catch (Exception e) { | ||
| throw new RuntimeException( | ||
| "Formatting failed in worker daemon. This may indicate a bug in palantir-java-format " | ||
| + "or insufficient resources. Original error: " + e.getMessage(), | ||
| e); | ||
| } | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -18,13 +18,15 @@ | |
| import com.diffplug.spotless.FileSignature; | ||
| import com.diffplug.spotless.FormatterFunc; | ||
| import com.diffplug.spotless.FormatterStep; | ||
| import com.palantir.javaformat.java.FormatterService; | ||
| import java.io.File; | ||
| import java.io.IOException; | ||
| import java.io.Serializable; | ||
| import java.io.UncheckedIOException; | ||
| import java.nio.file.Files; | ||
| import java.util.function.Supplier; | ||
| import org.gradle.api.artifacts.Configuration; | ||
| import org.gradle.workers.WorkQueue; | ||
| import org.gradle.workers.WorkerExecutor; | ||
|
|
||
| public final class PalantirJavaFormatStep { | ||
|
|
||
|
|
@@ -35,10 +37,10 @@ private PalantirJavaFormatStep() {} | |
| private static final String NAME = "palantir-java-format"; | ||
|
|
||
| /** Creates a step which formats everything - code, import order, and unused imports. */ | ||
| public static FormatterStep create(Configuration palantirJavaFormat, Supplier<FormatterService> memoizedService) { | ||
| public static FormatterStep create(Configuration palantirJavaFormat, WorkerExecutor workerExecutor) { | ||
| ensureImplementationNotDirectlyLoadable(); | ||
| return FormatterStep.createLazy( | ||
| NAME, () -> new State(palantirJavaFormat::getFiles, memoizedService), State::createFormat); | ||
| NAME, () -> new State(palantirJavaFormat::getFiles, workerExecutor), State::createFormat); | ||
| } | ||
|
|
||
| static final class State implements Serializable { | ||
|
|
@@ -65,19 +67,19 @@ static final class State implements Serializable { | |
|
|
||
| private final transient Supplier<Iterable<File>> jarsSupplier; | ||
|
|
||
| // Transient as this is not serializable. | ||
| private final transient Supplier<FormatterService> memoizedFormatter; | ||
| private final transient WorkerExecutor workerExecutor; | ||
|
|
||
| /** | ||
| * Build a cacheable state for spotless from the given jars, that uses the given {@link FormatterService}. | ||
| * Build a cacheable state for spotless from the given jars, using the Worker API for process isolation. | ||
| * | ||
| * @param jarsSupplier Supplies the jars that contain the palantir-java-format implementation. This is only used for caching and | ||
| * up-to-dateness purposes. | ||
| * @param workerExecutor WorkerExecutor for process isolation. | ||
| */ | ||
| @SuppressWarnings("for-rollout:NullAway") | ||
| State(Supplier<Iterable<File>> jarsSupplier, Supplier<FormatterService> memoizedFormatter) { | ||
| State(Supplier<Iterable<File>> jarsSupplier, WorkerExecutor workerExecutor) { | ||
| this.jarsSupplier = jarsSupplier; | ||
| this.memoizedFormatter = memoizedFormatter; | ||
| this.workerExecutor = workerExecutor; | ||
| } | ||
|
|
||
| @SuppressWarnings("NullableProblems") | ||
|
|
@@ -91,12 +93,55 @@ FormatterFunc createFormat() { | |
| // https://github.com/diffplug/spotless/blob/228eb10af382b19e130d8d9479f7a95238cb4358/lib/src/main/java/com/diffplug/spotless/FileSignature.java#L138-L143 | ||
| this.jarsSignature = FileSignature.signAsSet(jars); | ||
|
|
||
| return memoizedFormatter.get().formatSourceReflowStringsAndFixImports(input); | ||
| return formatWithWorker(input, jars); | ||
| } catch (IOException e) { | ||
| throw new UncheckedIOException(e); | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| private String formatWithWorker(String input, Iterable<File> jars) throws IOException { | ||
| File inputFile = | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this is the best way I could come-up with to get the formatted string in and out |
||
| Files.createTempFile("palantir-java-format-input-", ".java").toFile(); | ||
| File outputFile = Files.createTempFile("palantir-java-format-output-", ".java") | ||
| .toFile(); | ||
|
|
||
| try { | ||
| Files.writeString(inputFile.toPath(), input); | ||
|
|
||
| WorkQueue workQueue = workerExecutor.processIsolation(workerSpec -> { | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Hmm failure rate has increased to about 30% I wonder if this is just adding loads of overhead
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. As we are creating a temp io file and a new worker for every single file |
||
| workerSpec.getClasspath().from(jars); | ||
| workerSpec.forkOptions(options -> { | ||
| options.jvmArgs( | ||
| "--add-exports", | ||
| "jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", | ||
| "--add-exports", | ||
| "jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", | ||
| "--add-exports", | ||
| "jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", | ||
| "--add-exports", | ||
| "jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", | ||
| "--add-exports", | ||
| "jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED"); | ||
| }); | ||
| }); | ||
|
|
||
| workQueue.submit(FormatJavaWorkAction.class, parameters -> { | ||
| parameters.getInputFile().set(inputFile); | ||
| parameters.getOutputFile().set(outputFile); | ||
| parameters.getFormatterClasspath().from(jars); | ||
| }); | ||
|
|
||
| workQueue.await(); | ||
|
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. need to await to ensure formatting is done |
||
|
|
||
| return Files.readString(outputFile.toPath()); | ||
| } catch (Exception e) { | ||
| throw new RuntimeException("Formatting failed using Worker API", e); | ||
| } finally { | ||
| inputFile.delete(); | ||
| outputFile.delete(); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static void ensureImplementationNotDirectlyLoadable() { | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Running this locally, does it log this per file or just once per gradle invocation? Not sure is gradle is doing something funky with classloaders even in process isolation that means this may be new each time.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
once per gradle invocation - or at least per spotless invocation: