Use clang-format as the default sketch formatter.

- Bumped `clangd` to `14.0.0`,
 - Can use `.clang-format` from:
   - current sketch folder,
   - `~/.arduinoIDE/.clang-format`,
   - `directories#data/.clang-format`, or
   - falls back to default formatter styles.

Closes #1009
Closes #566

Signed-off-by: Akos Kitta <a.kitta@arduino.cc>

arduino-ide-extension/package.json

@@ -163,7 +163,7 @@
       "version": "2.0.0"
     },
     "clangd": {
-      "version": "13.0.0"
+      "version": "14.0.0"
     },
     "languageServer": {
       "version": "0.6.0"

arduino-ide-extension/scripts/download-ls.js

@@ -66,21 +66,24 @@
     build,
     `arduino-language-server${platform === 'win32' ? '.exe' : ''}`
   );
-  let clangdExecutablePath, lsSuffix, clangdSuffix;
+  let clangdExecutablePath, clangFormatExecutablePath, lsSuffix, clangdSuffix;
 
   switch (platformArch) {
     case 'darwin-x64':
       clangdExecutablePath = path.join(build, 'clangd');
+      clangFormatExecutablePath = path.join(build, 'clang-format');
       lsSuffix = 'macOS_64bit.tar.gz';
       clangdSuffix = 'macOS_64bit';
       break;
     case 'linux-x64':
       clangdExecutablePath = path.join(build, 'clangd');
+      clangFormatExecutablePath = path.join(build, 'clang-format');
       lsSuffix = 'Linux_64bit.tar.gz';
       clangdSuffix = 'Linux_64bit';
       break;
     case 'win32-x64':
       clangdExecutablePath = path.join(build, 'clangd.exe');
+      clangFormatExecutablePath = path.join(build, 'clang-format.exe');
       lsSuffix = 'Windows_64bit.zip';
       clangdSuffix = 'Windows_64bit';
       break;
@@ -103,4 +106,15 @@
   downloader.downloadUnzipAll(clangdUrl, build, clangdExecutablePath, force, {
     strip: 1,
   }); // `strip`: the new clangd (12.x) is zipped into a folder, so we have to strip the outmost folder.
+
+  const clangdFormatUrl = `https://downloads.arduino.cc/tools/clang-format_${clangdVersion}_${clangdSuffix}.tar.bz2`;
+  downloader.downloadUnzipAll(
+    clangdFormatUrl,
+    build,
+    clangFormatExecutablePath,
+    force,
+    {
+      strip: 1,
+    }
+  );
 })();

arduino-ide-extension/src/browser/arduino-ide-frontend-module.ts

@@ -280,6 +280,10 @@ import { EditorManager } from './theia/editor/editor-manager';
 import { HostedPluginEvents } from './hosted-plugin-events';
 import { HostedPluginSupport } from './theia/plugin-ext/hosted-plugin';
 import { HostedPluginSupport as TheiaHostedPluginSupport } from '@theia/plugin-ext/lib/hosted/browser/hosted-plugin';
+import { Formatter, FormatterPath } from '../common/protocol/formatter';
+import { Format } from './contributions/format';
+import { MonacoFormattingConflictsContribution } from './theia/monaco/monaco-formatting-conflicts';
+import { MonacoFormattingConflictsContribution as TheiaMonacoFormattingConflictsContribution } from '@theia/monaco/lib/browser/monaco-formatting-conflicts';
 
 const ElementQueries = require('css-element-queries/src/ElementQueries');
 
@@ -573,6 +577,12 @@ export default new ContainerModule((bind, unbind, isBound, rebind) => {
     )
     .inSingletonScope();
 
+  bind(Formatter)
+    .toDynamicValue(({ container }) =>
+      WebSocketConnectionProvider.createProxy(container, FormatterPath)
+    )
+    .inSingletonScope();
+
   bind(ArduinoFirmwareUploader)
     .toDynamicValue((context) =>
       WebSocketConnectionProvider.createProxy(
@@ -640,6 +650,14 @@ export default new ContainerModule((bind, unbind, isBound, rebind) => {
   Contribution.configure(bind, ArchiveSketch);
   Contribution.configure(bind, AddZipLibrary);
   Contribution.configure(bind, PlotterFrontendContribution);
+  Contribution.configure(bind, Format);
+
+  // Disabled the quick-pick customization from Theia when multiple formatters are available.
+  // Use the default VS Code behavior, and pick the first one. In the IDE2, clang-format has `exclusive` selectors.
+  bind(MonacoFormattingConflictsContribution).toSelf().inSingletonScope();
+  rebind(TheiaMonacoFormattingConflictsContribution).toService(
+    MonacoFormattingConflictsContribution
+  );
 
   bind(ResponseServiceImpl)
     .toSelf()

arduino-ide-extension/src/browser/contributions/format.ts

@@ -0,0 +1,94 @@
+import { MaybePromise } from '@theia/core';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import * as monaco from '@theia/monaco-editor-core';
+import { Formatter } from '../../common/protocol/formatter';
+import { Contribution, URI } from './contribution';
+
+@injectable()
+export class Format
+  extends Contribution
+  implements
+    monaco.languages.DocumentRangeFormattingEditProvider,
+    monaco.languages.DocumentFormattingEditProvider
+{
+  @inject(Formatter)
+  private readonly formatter: Formatter;
+
+  override onStart(): MaybePromise<void> {
+    const selector = this.selectorOf('ino', 'c', 'cpp', 'h', 'hpp', 'pde');
+    monaco.languages.registerDocumentRangeFormattingEditProvider(
+      selector,
+      this
+    );
+    monaco.languages.registerDocumentFormattingEditProvider(selector, this);
+  }
+  async provideDocumentRangeFormattingEdits(
+    model: monaco.editor.ITextModel,
+    range: monaco.Range,
+    options: monaco.languages.FormattingOptions,
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    _token: monaco.CancellationToken
+  ): Promise<monaco.languages.TextEdit[]> {
+    const text = await this.format(model, range, options);
+    return [{ range, text }];
+  }
+
+  async provideDocumentFormattingEdits(
+    model: monaco.editor.ITextModel,
+    options: monaco.languages.FormattingOptions,
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    _token: monaco.CancellationToken
+  ): Promise<monaco.languages.TextEdit[]> {
+    const range = this.fullRange(model);
+    const text = await this.format(model, range, options);
+    return [{ range, text }];
+  }
+
+  private fullRange(model: monaco.editor.ITextModel): monaco.Range {
+    const lastLine = model.getLineCount();
+    const lastLineMaxColumn = model.getLineMaxColumn(lastLine);
+    const end = new monaco.Position(lastLine, lastLineMaxColumn);
+    return monaco.Range.fromPositions(new monaco.Position(1, 1), end);
+  }
+
+  /**
+   * From the currently opened workspaces (IDE2 has always one), it calculates all possible
+   * folder locations where the `.clang-format` file could be.
+   */
+  private formatterConfigFolderUris(model: monaco.editor.ITextModel): string[] {
+    const editorUri = new URI(model.uri.toString());
+    return this.workspaceService
+      .tryGetRoots()
+      .map(({ resource }) => resource)
+      .filter((workspaceUri) => workspaceUri.isEqualOrParent(editorUri))
+      .map((uri) => uri.toString());
+  }
+
+  private format(
+    model: monaco.editor.ITextModel,
+    range: monaco.Range,
+    options: monaco.languages.FormattingOptions
+  ): Promise<string> {
+    console.info(
+      `Formatting ${model.uri.toString()} [Range: ${JSON.stringify(
+        range.toJSON()
+      )}]`
+    );
+    const content = model.getValueInRange(range);
+    const formatterConfigFolderUris = this.formatterConfigFolderUris(model);
+    return this.formatter.format({
+      content,
+      formatterConfigFolderUris,
+      options,
+    });
+  }
+
+  private selectorOf(
+    ...languageId: string[]
+  ): monaco.languages.LanguageSelector {
+    return languageId.map((language) => ({
+      language,
+      exclusive: true, // <-- this should make sure the custom formatter has higher precedence over the LS formatter.
+    }));
+  }
+}

arduino-ide-extension/src/browser/theia/monaco/monaco-formatting-conflicts.ts

@@ -0,0 +1,12 @@
+import { injectable } from '@theia/core/shared/inversify';
+import { MonacoFormattingConflictsContribution as TheiaMonacoFormattingConflictsContribution } from '@theia/monaco/lib/browser/monaco-formatting-conflicts';
+
+@injectable()
+export class MonacoFormattingConflictsContribution extends TheiaMonacoFormattingConflictsContribution {
+  override async initialize(): Promise<void> {
+    // NOOP - does not register a custom formatting conflicts selects.
+    // Does not get and set formatter preferences when selecting from multiple formatters.
+    // Does not show quick-pick input when multiple formatters are available for the text model.
+    // Uses the default behavior from VS Code: https://github.com/microsoft/vscode/blob/fb9f488e51af2e2efe95a34f24ca11e1b2a3f744/src/vs/editor/editor.api.ts#L19-L21
+  }
+}

arduino-ide-extension/src/common/protocol/formatter.ts

@@ -0,0 +1,23 @@
+export const FormatterPath = '/services/formatter';
+export const Formatter = Symbol('Formatter');
+export interface Formatter {
+  format({
+    content,
+    formatterConfigFolderUris,
+    options,
+  }: {
+    content: string;
+    formatterConfigFolderUris: string[];
+    options?: FormatterOptions;
+  }): Promise<string>;
+}
+export interface FormatterOptions {
+  /**
+   * Size of a tab in spaces.
+   */
+  tabSize: number;
+  /**
+   * Prefer spaces over tabs.
+   */
+  insertSpaces: boolean;
+}

arduino-ide-extension/src/node/arduino-ide-backend-module.ts

@@ -94,6 +94,8 @@ import WebSocketServiceImpl from './web-socket/web-socket-service-impl';
 import { WebSocketService } from './web-socket/web-socket-service';
 import { ArduinoLocalizationContribution } from './arduino-localization-contribution';
 import { LocalizationContribution } from '@theia/core/lib/node/i18n/localization-contribution';
+import { ClangFormatter } from './clang-formatter';
+import { FormatterPath } from '../common/protocol/formatter';
 
 export default new ContainerModule((bind, unbind, isBound, rebind) => {
   bind(BackendApplication).toSelf().inSingletonScope();
@@ -126,6 +128,17 @@ export default new ContainerModule((bind, unbind, isBound, rebind) => {
     )
     .inSingletonScope();
 
+  // Shared formatter
+  bind(ClangFormatter).toSelf().inSingletonScope();
+  bind(ConnectionHandler)
+    .toDynamicValue(
+      ({ container }) =>
+        new JsonRpcConnectionHandler(FormatterPath, () =>
+          container.get(ClangFormatter)
+        )
+    )
+    .inSingletonScope();
+
   // Examples service. One per backend, each connected FE gets a proxy.
   bind(ConnectionContainerModule).toConstantValue(
     ConnectionContainerModule.create(({ bind, bindBackendService }) => {

arduino-ide-extension/src/node/clang-formatter.ts

@@ -0,0 +1,279 @@
+import { EnvVariablesServer } from '@theia/core/lib/common/env-variables';
+import { MaybePromise } from '@theia/core/lib/common/types';
+import { FileUri } from '@theia/core/lib/node/file-uri';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { constants, promises as fs } from 'fs';
+import { join } from 'path';
+import { ConfigService } from '../common/protocol';
+import { Formatter, FormatterOptions } from '../common/protocol/formatter';
+import { getExecPath, spawnCommand } from './exec-util';
+
+@injectable()
+export class ClangFormatter implements Formatter {
+  @inject(ConfigService)
+  private readonly configService: ConfigService;
+
+  @inject(EnvVariablesServer)
+  private readonly envVariableServer: EnvVariablesServer;
+
+  async format({
+    content,
+    formatterConfigFolderUris,
+    options,
+  }: {
+    content: string;
+    formatterConfigFolderUris: string[];
+    options?: FormatterOptions;
+  }): Promise<string> {
+    const [execPath, style] = await Promise.all([
+      this.execPath(),
+      this.style(formatterConfigFolderUris, options),
+    ]);
+    const formatted = await spawnCommand(
+      `"${execPath}"`,
+      [style],
+      console.error,
+      content
+    );
+    return formatted;
+  }
+
+  private _execPath: string | undefined;
+  private async execPath(): Promise<string> {
+    if (this._execPath) {
+      return this._execPath;
+    }
+    this._execPath = await getExecPath('clang-format');
+    return this._execPath;
+  }
+
+  /**
+   * Calculates the `-style` flag for the formatter. Uses a `.clang-format` file if exists.
+   * Otherwise, falls back to the default config.
+   *
+   * Style precedence:
+   *  1. in the sketch folder,
+   *  1. `~/.arduinoIDE/.clang-format`,
+   *  1. `directories#data/.clang-format`, and
+   *  1. default style flag as a string.
+   *
+   * See: https://github.com/arduino/arduino-ide/issues/566
+   */
+  private async style(
+    formatterConfigFolderUris: string[],
+    options?: FormatterOptions
+  ): Promise<string> {
+    const clangFormatPaths = await Promise.all([
+      ...formatterConfigFolderUris.map((uri) => this.clangConfigPath(uri)),
+      this.clangConfigPath(this.configDirPath()),
+      this.clangConfigPath(this.dataDirPath()),
+    ]);
+    const first = clangFormatPaths.filter(Boolean).shift();
+    if (first) {
+      console.debug(
+        `Using ${ClangFormatFile} style configuration from '${first}'.`
+      );
+      return `-style=file:"${first}"`;
+    }
+    return `-style="${style(toClangOptions(options))}"`;
+  }
+
+  private async dataDirPath(): Promise<string> {
+    const { dataDirUri } = await this.configService.getConfiguration();
+    return FileUri.fsPath(dataDirUri);
+  }
+
+  private async configDirPath(): Promise<string> {
+    const configDirUri = await this.envVariableServer.getConfigDirUri();
+    return FileUri.fsPath(configDirUri);
+  }
+
+  private async clangConfigPath(
+    folderUri: MaybePromise<string>
+  ): Promise<string | undefined> {
+    const folderPath = FileUri.fsPath(await folderUri);
+    const clangFormatPath = join(folderPath, ClangFormatFile);
+    try {
+      await fs.access(clangFormatPath, constants.R_OK);
+      return clangFormatPath;
+    } catch {
+      return undefined;
+    }
+  }
+}
+
+interface ClangFormatOptions {
+  readonly UseTab: 'Never' | 'ForIndentation';
+  readonly TabWidth: number;
+}
+
+const ClangFormatFile = '.clang-format';
+
+function toClangOptions(
+  options?: FormatterOptions | undefined
+): ClangFormatOptions {
+  if (!!options) {
+    return {
+      UseTab: options.insertSpaces ? 'Never' : 'ForIndentation',
+      TabWidth: options.tabSize,
+    };
+  }
+  return { UseTab: 'Never', TabWidth: 2 };
+}
+
+// See: https://releases.llvm.org/11.0.1/tools/clang/docs/ClangFormatStyleOptions.html
+export function style({ TabWidth, UseTab }: ClangFormatOptions): string {
+  return JSON.stringify(styleJson({ TabWidth, UseTab })).replace(/\"/g, '\\"');
+}
+
+function styleJson({
+  TabWidth,
+  UseTab,
+}: ClangFormatOptions): Record<string, unknown> {
+  return {
+    Language: 'Cpp',
+    // # LLVM is the default style setting, used when a configuration option is not set here
+    BasedOnStyle: 'LLVM',
+    AccessModifierOffset: -2,
+    AlignAfterOpenBracket: 'Align',
+    AlignConsecutiveAssignments: false,
+    AlignConsecutiveBitFields: false,
+    AlignConsecutiveDeclarations: false,
+    AlignConsecutiveMacros: false,
+    AlignEscapedNewlines: 'DontAlign',
+    AlignOperands: 'Align',
+    AlignTrailingComments: true,
+    AllowAllArgumentsOnNextLine: true,
+    AllowAllConstructorInitializersOnNextLine: true,
+    AllowAllParametersOfDeclarationOnNextLine: true,
+    AllowShortBlocksOnASingleLine: 'Always',
+    AllowShortCaseLabelsOnASingleLine: true,
+    AllowShortEnumsOnASingleLine: true,
+    AllowShortFunctionsOnASingleLine: 'Empty',
+    AllowShortIfStatementsOnASingleLine: 'Always',
+    AllowShortLambdasOnASingleLine: 'Empty',
+    AllowShortLoopsOnASingleLine: true,
+    AlwaysBreakAfterDefinitionReturnType: 'None',
+    AlwaysBreakAfterReturnType: 'None',
+    AlwaysBreakBeforeMultilineStrings: false,
+    AlwaysBreakTemplateDeclarations: 'No',
+    BinPackArguments: true,
+    BinPackParameters: true,
+    // # Only used when "BreakBeforeBraces" set to "Custom"
+    BraceWrapping: {
+      AfterCaseLabel: false,
+      AfterClass: false,
+      AfterControlStatement: 'Never',
+      AfterEnum: false,
+      AfterFunction: false,
+      AfterNamespace: false,
+      // #AfterObjCDeclaration:
+      AfterStruct: false,
+      AfterUnion: false,
+      AfterExternBlock: false,
+      BeforeCatch: false,
+      BeforeElse: false,
+      BeforeLambdaBody: false,
+      BeforeWhile: false,
+      IndentBraces: false,
+      SplitEmptyFunction: false,
+      SplitEmptyRecord: false,
+      SplitEmptyNamespace: false,
+    },
+    // # Java-specific
+    // #BreakAfterJavaFieldAnnotations:
+    BreakBeforeBinaryOperators: 'NonAssignment',
+    BreakBeforeBraces: 'Attach',
+    BreakBeforeTernaryOperators: true,
+    BreakConstructorInitializers: 'BeforeColon',
+    BreakInheritanceList: 'BeforeColon',
+    BreakStringLiterals: false,
+    ColumnLimit: 0,
+    // # "" matches none
+    CommentPragmas: '',
+    CompactNamespaces: false,
+    ConstructorInitializerAllOnOneLineOrOnePerLine: true,
+    ConstructorInitializerIndentWidth: 2,
+    ContinuationIndentWidth: 2,
+    Cpp11BracedListStyle: false,
+    DeriveLineEnding: true,
+    DerivePointerAlignment: true,
+    DisableFormat: false,
+    // # Docs say "Do not use this in config files". The default (LLVM 11.0.1) is "false".
+    // #ExperimentalAutoDetectBinPacking:
+    FixNamespaceComments: false,
+    ForEachMacros: [],
+    IncludeBlocks: 'Preserve',
+    IncludeCategories: [],
+    // # "" matches none
+    IncludeIsMainRegex: '',
+    IncludeIsMainSourceRegex: '',
+    IndentCaseBlocks: true,
+    IndentCaseLabels: true,
+    IndentExternBlock: 'Indent',
+    IndentGotoLabels: false,
+    IndentPPDirectives: 'None',
+    IndentWidth: 2,
+    IndentWrappedFunctionNames: false,
+    InsertTrailingCommas: 'None',
+    // # Java-specific
+    // #JavaImportGroups:
+    // # JavaScript-specific
+    // #JavaScriptQuotes:
+    // #JavaScriptWrapImports
+    KeepEmptyLinesAtTheStartOfBlocks: true,
+    MacroBlockBegin: '',
+    MacroBlockEnd: '',
+    // # Set to a large number to effectively disable
+    MaxEmptyLinesToKeep: 100000,
+    NamespaceIndentation: 'None',
+    NamespaceMacros: [],
+    // # Objective C-specific
+    // #ObjCBinPackProtocolList:
+    // #ObjCBlockIndentWidth:
+    // #ObjCBreakBeforeNestedBlockParam:
+    // #ObjCSpaceAfterProperty:
+    // #ObjCSpaceBeforeProtocolList
+    PenaltyBreakAssignment: 1,
+    PenaltyBreakBeforeFirstCallParameter: 1,
+    PenaltyBreakComment: 1,
+    PenaltyBreakFirstLessLess: 1,
+    PenaltyBreakString: 1,
+    PenaltyBreakTemplateDeclaration: 1,
+    PenaltyExcessCharacter: 1,
+    PenaltyReturnTypeOnItsOwnLine: 1,
+    // # Used as a fallback if alignment style can't be detected from code (DerivePointerAlignment: true)
+    PointerAlignment: 'Right',
+    RawStringFormats: [],
+    ReflowComments: false,
+    SortIncludes: false,
+    SortUsingDeclarations: false,
+    SpaceAfterCStyleCast: false,
+    SpaceAfterLogicalNot: false,
+    SpaceAfterTemplateKeyword: false,
+    SpaceBeforeAssignmentOperators: true,
+    SpaceBeforeCpp11BracedList: false,
+    SpaceBeforeCtorInitializerColon: true,
+    SpaceBeforeInheritanceColon: true,
+    SpaceBeforeParens: 'ControlStatements',
+    SpaceBeforeRangeBasedForLoopColon: true,
+    SpaceBeforeSquareBrackets: false,
+    SpaceInEmptyBlock: false,
+    SpaceInEmptyParentheses: false,
+    SpacesBeforeTrailingComments: 2,
+    SpacesInAngles: false,
+    SpacesInCStyleCastParentheses: false,
+    SpacesInConditionalStatement: false,
+    SpacesInContainerLiterals: false,
+    SpacesInParentheses: false,
+    SpacesInSquareBrackets: false,
+    Standard: 'Auto',
+    StatementMacros: [],
+    TabWidth,
+    TypenameMacros: [],
+    // # Default to LF if line endings can't be detected from the content (DeriveLineEnding).
+    UseCRLF: false,
+    UseTab,
+    WhitespaceSensitiveMacros: [],
+  };
+}

arduino-ide-extension/src/node/exec-util.ts

@@ -47,7 +47,8 @@ export async function getExecPath(
 export function spawnCommand(
   command: string,
   args: string[],
-  onError: (error: Error) => void = (error) => console.log(error)
+  onError: (error: Error) => void = (error) => console.log(error),
+  stdIn?: string
 ): Promise<string> {
   return new Promise<string>((resolve, reject) => {
     const cp = spawn(command, args, { windowsHide: true, shell: true });
@@ -87,5 +88,9 @@ export function spawnCommand(
         return;
       }
     });
+    if (stdIn !== undefined) {
+      cp.stdin.write(stdIn);
+      cp.stdin.end();
+    }
   });
 }