From 625a24ac2659909303556fd77820325c069031e0 Mon Sep 17 00:00:00 2001 From: Nicolas Adment <39568358+nadment@users.noreply.github.com> Date: Mon, 16 Nov 2026 20:08:25 +0100 Subject: [PATCH] Add SharePoint VFS plugin to read and write files in SharePoint and Teams through Microsoft Graph, fixes #9531 --- assemblies/plugins/pom.xml | 6 + .../metadata/api/HopMetadataPropertyType.java | 1 + docs/hop-user-manual/modules/ROOT/nav.adoc | 2 + .../ROOT/pages/metadata-types/index.adoc | 1 + .../metadata-types/sharepoint-connection.adoc | 74 +++ .../modules/ROOT/pages/vfs.adoc | 2 + .../ROOT/pages/vfs/sharepoint-vfs.adoc | 88 +++ plugins/tech/microsoft365/pom.xml | 263 +++++++++ .../microsoft365/src/assembly/assembly.xml | 37 ++ .../vfs/sharepoint/SharePointFileName.java | 135 +++++ .../sharepoint/SharePointFileNameParser.java | 71 +++ .../vfs/sharepoint/SharePointFileObject.java | 318 +++++++++++ .../sharepoint/SharePointFileProvider.java | 71 +++ .../vfs/sharepoint/SharePointFileSystem.java | 503 ++++++++++++++++++ .../vfs/sharepoint/SharePointVfsPlugin.java | 82 +++ .../metadatatype/SharePointConnection.java | 224 ++++++++ .../SharePointConnectionEditor.java | 234 ++++++++ .../messages/messages_en_US.properties | 38 ++ .../src/main/resources/sharepoint.svg | 42 ++ .../src/main/resources/version.xml | 20 + .../sharepoint/SharePointFileNameTest.java | 112 ++++ plugins/tech/pom.xml | 1 + 22 files changed, 2325 insertions(+) create mode 100644 docs/hop-user-manual/modules/ROOT/pages/metadata-types/sharepoint-connection.adoc create mode 100644 docs/hop-user-manual/modules/ROOT/pages/vfs/sharepoint-vfs.adoc create mode 100644 plugins/tech/microsoft365/pom.xml create mode 100644 plugins/tech/microsoft365/src/assembly/assembly.xml create mode 100644 plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileName.java create mode 100644 plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileNameParser.java create mode 100644 plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileObject.java create mode 100644 plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileProvider.java create mode 100644 plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileSystem.java create mode 100644 plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointVfsPlugin.java create mode 100644 plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnection.java create mode 100644 plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnectionEditor.java create mode 100644 plugins/tech/microsoft365/src/main/resources/org/apache/hop/vfs/sharepoint/metadatatype/messages/messages_en_US.properties create mode 100644 plugins/tech/microsoft365/src/main/resources/sharepoint.svg create mode 100644 plugins/tech/microsoft365/src/main/resources/version.xml create mode 100644 plugins/tech/microsoft365/src/test/java/org/apache/hop/vfs/sharepoint/SharePointFileNameTest.java diff --git a/assemblies/plugins/pom.xml b/assemblies/plugins/pom.xml index 6db4ddd8280..805d9f7b2ba 100644 --- a/assemblies/plugins/pom.xml +++ b/assemblies/plugins/pom.xml @@ -1839,6 +1839,12 @@ ${project.version} zip + + org.apache.hop + hop-tech-microsoft365 + ${project.version} + zip + org.apache.hop hop-tech-minio diff --git a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java index cb324716c6f..a2a41a8e56a 100644 --- a/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java +++ b/core/src/main/java/org/apache/hop/metadata/api/HopMetadataPropertyType.java @@ -128,6 +128,7 @@ public enum HopMetadataPropertyType { VFS_FTP_CONNECTION, VFS_HDFS_CONNECTION, VFS_SMB_CONNECTION, + VFS_SHAREPOINT_CONNECTION, VFS_GIT_CONNECTION, // DATA STREAM diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index 09e95922d4a..e768210ab85 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -476,6 +476,7 @@ under the License. * xref:vfs.adoc[Virtual File System] ** xref:vfs/aws-s3-vfs.adoc[Amazon Web Services S3] ** xref:vfs/azure-blob-storage-vfs.adoc[Microsoft Azure Storage] +** xref:vfs/sharepoint-vfs.adoc[Microsoft SharePoint] ** xref:vfs/databricks-vfs.adoc[Databricks Volumes / Workspace] ** xref:vfs/dropbox-vfs.adoc[Dropbox] ** xref:vfs/google-cloud-storage-vfs.adoc[Google Cloud Storage] @@ -564,6 +565,7 @@ under the License. ** xref:metadata-types/spark-catalog.adoc[Lakehouse catalog] ** xref:metadata-types/mail-server-connection.adoc[Mail Server Connection] ** xref:metadata-types/masking-pattern.adoc[Masking Pattern] +** xref:metadata-types/sharepoint-connection.adoc[SharePoint Connection] ** xref:metadata-types/minio-connection.adoc[Minio (S3) Connection] ** xref:metadata-types/mongodb-connection.adoc[MongoDB Connection] ** xref:metadata-types/graphs/neo4j-connection.adoc[Neo4j Connection] diff --git a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/index.adoc b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/index.adoc index 4bd3e91015b..8852b2ebcf5 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/index.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/index.adoc @@ -57,6 +57,7 @@ xref:metadata-types/data-stream/data-stream.adoc[Data Stream]: A Data Stream can * xref:metadata-types/mongodb-connection.adoc[MongoDB Connection]: Describes a MongoDB connection * xref:metadata-types/mail-server-connection.adoc[Mail Server Connection]: Describes a mail server connection * xref:metadata-types/masking-pattern.adoc[Masking Pattern]: A reusable rule for replacing personal data in a field +* xref:metadata-types/sharepoint-connection.adoc[SharePoint Connection]: Named VFS access to the document libraries of the SharePoint sites of a Microsoft 365 * xref:metadata-types/minio-connection.adoc[Minio (S3) Connection]: Connect to one or more S3 endpoints using a Minio client * xref:metadata-types/naming-scheme.adoc[Naming Scheme]: Reusable rules to normalize Hop field names, database identifiers, and file or folder names * xref:metadata-types/webdav-connection.adoc[WebDAV Connection]: Named WebDAV or WebDAV-over-HTTPS endpoint used as a VFS scheme in file paths diff --git a/docs/hop-user-manual/modules/ROOT/pages/metadata-types/sharepoint-connection.adoc b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/sharepoint-connection.adoc new file mode 100644 index 00000000000..d6dbceb8655 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/metadata-types/sharepoint-connection.adoc @@ -0,0 +1,74 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:imagesdir: ../../assets/images/ +:page-pagination: +:description: A named file system on the document libraries of Microsoft SharePoint sites, through the Microsoft Graph API + += SharePoint Connection + +== Description + +This metadata type adds the SharePoint sites of a Microsoft 365 tenant to your project as a file system, and describes how Apache Hop signs in to that tenant through the Microsoft Graph API. +The name of the connection is also a file system scheme: the SharePoint sites of the tenant, their document libraries and the files in them can be read and written with paths such as `+name:///site/library/folder/file.csv+`. +See xref:vfs/sharepoint-vfs.adoc[Microsoft SharePoint VFS] for the layout of these paths. + +Hop signs in as an application, not as a person. +Register an application in Microsoft Entra ID, grant it Microsoft Graph *application* permissions and have an administrator consent to them: + +* `Sites.Read.All` to read files +* `Sites.ReadWrite.All` to also create, replace, rename and delete files and folders +* `Sites.Selected` to only reach the sites the application was granted. These sites can not be listed: use the full path of the site, `+name:///site/library+`, and set the SharePoint host on the connection if the site is not found. + +== Options + +[options="header"] +|=== +|Option |Description +|Name| Name of the connection, this is the scheme used in file paths: `name://` +|Description| Description field for additional information +|Authentication type a| How Hop signs in to Microsoft Entra ID: + +* *Client secret authentication*: the client ID and a client secret of the registered application +* *Client certificate authentication*: the client ID and a certificate of the registered application +* *Managed identity*: the identity of the environment Hop runs in, found through the default Azure credential chain (managed identity, environment variables, Azure CLI, and so on) +|Tenant ID| The Directory (tenant) ID of your Microsoft Entra tenant +|SharePoint host| The SharePoint host of the tenant, for example `contoso.sharepoint.com`. Optional in most cases, see <<_sharepoint_host>>. +|Client ID| The Application (client) ID of the registered application. Not used with a managed identity. +|Client Secret| The client secret of the registered application, for client secret authentication +|PEM Certificate| The certificate and its private key in PEM format, for client certificate authentication +|=== + +All the options except the name, the description and the authentication type accept variables. + +=== SharePoint host + +The host is only used to find a site that is not in the list of the sites of the tenant, typically because the application is not allowed to list them. +Such a site is read by its address, `/sites/` or `/teams/`. + +When the option is empty, Hop reads the root site of the tenant to find the host, so the connection usually works without it. +Set it when: + +* The application can not read the root site, for example with `Sites.Selected` and no grant on the root site. +The log then says `The SharePoint host of the tenant can not be read, set it on the connection`. +* The sites are on another host than the root site of the tenant. +* You want to save the request that reads the root site. + +When the application can list the sites (`Sites.Read.All` or `Sites.ReadWrite.All`), the sites are found by their name and the host is not used. + +**Test** on this editor requests a Microsoft Graph token with the values in the form. + +**Explore** on this editor opens the xref:hop-gui/vfs-file-explorer.adoc[VFS File Explorer] at `+name:///+`. diff --git a/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc b/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc index e7f49ce5662..c8376306547 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/vfs.adoc @@ -52,6 +52,7 @@ Click the File system name to access more detailed file system documentation. |xref:vfs/google-drive-vfs.adoc[Google Drive]|Provides access to Google Drive folders|`googledrive://` |xref:vfs/git-vfs.adoc[Git]|Named connection to a git repository checked out at a branch, tag or commit (marketplace plugin `hop-tech-git-vfs`)|`+:///path+` |xref:vfs/hdfs-vfs.adoc[HDFS]|Named connection to HDFS via HttpFS, Knox or WebHDFS (marketplace plugin `hop-tech-hadoop`)|`+:///path+` +|xref:vfs/sharepoint-vfs.adoc[Microsoft SharePoint]|Named xref:metadata-types/sharepoint-connection.adoc[SharePoint Connection] to the document libraries of SharePoint sites|`+:///site/library/path+` |xref:vfs/smb-vfs.adoc[SMB]|Named connection to an SMB2/SMB3 file share|`+:///path+` |xref:metadata-types/minio-connection.adoc[Minio connection]|Provides access to S3-compatible endpoints via a named Minio connection (metadata)|`+://+` |xref:metadata-types/webdav-connection.adoc[WebDAV Connection]|Provides access to WebDAV servers, either via the static schemes or a named connection (metadata)|`webdav4://`, `webdav4s://`, or `+://+` @@ -330,6 +331,7 @@ Server-side restrictions (read-only mounts, bucket policies, account permissions |`++` |✓|✓|✗|✓|✓|✓|✗|✗|read |`++` (see below) |✓|✓|✓|✓|✓|✓|✗|✗|read |`++` |✓|✓|✗|✓|✓|✓|✗|✗|read +|`++` |✓|✓|✗|✓|✓|✓|✗|✗|read |`++` |✓|✓|✓|✓|✓|✓|✓|✓|read |`webdav4`, `webdav4s`, named WebDAV connection |✓|✓|✗|✓|✓|✓|✓|✗|read diff --git a/docs/hop-user-manual/modules/ROOT/pages/vfs/sharepoint-vfs.adoc b/docs/hop-user-manual/modules/ROOT/pages/vfs/sharepoint-vfs.adoc new file mode 100644 index 00000000000..6e474b43403 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/vfs/sharepoint-vfs.adoc @@ -0,0 +1,88 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// + +:documentationPath: /vfs/ +:language: en_US +:description: Apache Hop supports reading from and writing to Microsoft SharePoint document libraries from almost anywhere in the platform through Apache VFS + += Microsoft SharePoint VFS + +== Scheme + +There is no fixed scheme. +Every xref:metadata-types/sharepoint-connection.adoc[SharePoint Connection] registers a scheme with its own name, so a connection named `m365` gives you: + +`**m365://**` + +The credentials are stored in the connection, never in the path. + +== Configuration + +Create a xref:metadata-types/sharepoint-connection.adoc[SharePoint Connection] in the metadata perspective. +The application it signs in with needs the Microsoft Graph application permission `Sites.Read.All` to read files, or `Sites.ReadWrite.All` to also write them. + +== Usage and testing + +The path has three parts: the site, the document library, and the path of the file or folder in that library. + +`+m365://////+` + +[options="header",cols="2,3"] +|=== +|Path|What it is +|`+m365:///+`|The SharePoint sites of the tenant +|`+m365:///Finance+`|The document libraries of the site `Finance` +|`+m365:///Finance/Documents+`|The root folder of the library `Documents` +|`+m365:///Finance/Documents/reports/2026/daily.csv+`|A file in that library +|=== + +* The site is the name of the site as it appears in its URL (`Finance` for `+https://contoso.sharepoint.com/sites/Finance+`). +* The library is the name of the document library. +The default library of a site is named `Documents` on an English tenant, even though its URL says `Shared Documents`. +* Site and library names are not case-sensitive. +When two sites have the same name, only the first one is reachable and the other one is reported in the log. +* With the permission `Sites.Selected` the root `+m365:///+` is empty, because the application can not list the sites. Set the SharePoint host on the connection and enter the path of a site it was granted, such as `+m365:///Finance+`. + +You can browse the sites, libraries and files in the xref:hop-gui/vfs-file-explorer.adoc[VFS File Explorer] or in any file dialog. + +== Microsoft Teams channels + +The files of a Microsoft Teams team are stored in a SharePoint site, so they are reached with the same paths. +Where the files of a channel are depends on the type of the channel. + +[options="header",cols="1,2,3"] +|=== +|Channel|Site|Path +|Standard|The site of the team|`+m365:////Documents//+` +|Private or shared|A site of its own, usually named `-`|`+m365:///-/Documents//+` +|=== + +* A channel is a folder of the library `Documents`, never a site or a library by itself. +The folder of the default channel is named `General`. +* The name of the site of a team is not always the name of the team. +To find the exact names, open the *Files* tab of the channel in Teams and choose *Open in SharePoint*: the site is what follows `/sites/` or `/teams/` in the URL, the folder is what follows `Shared Documents/`. +* Do not put `/sites` or `/teams` in the path, only the name of the site. +* With the permission `Sites.Selected`, the site of a private or shared channel has to be granted to the application separately from the site of the team. + +== Limitations + +* Sites and document libraries can be listed but not created, renamed or deleted. +* Appending to a file is not supported. +* A file or folder can only be renamed or moved inside its document library. +Moving it to another library or site copies the content and deletes the original. +* A file is buffered in a local temporary file while it is written and is sent to SharePoint when it is closed. +Files larger than 4 MB are sent in slices through an upload session. diff --git a/plugins/tech/microsoft365/pom.xml b/plugins/tech/microsoft365/pom.xml new file mode 100644 index 00000000000..a2860927cfb --- /dev/null +++ b/plugins/tech/microsoft365/pom.xml @@ -0,0 +1,263 @@ + + + + 4.0.0 + + + org.apache.hop + hop-plugins-tech + 2.21.0-SNAPSHOT + + + hop-tech-microsoft365 + jar + Hop Plugins Technology Microsoft 365 + + + 3.3.0 + 6.58.0 + + + + + + io.netty + netty-bom + ${netty.version} + pom + import + + + com.azure + azure-sdk-bom + ${azure-sdk-bom.version} + pom + import + + + org.apache.hop + hop-libs + ${project.version} + pom + import + + + + + + + com.microsoft.graph + microsoft-graph + ${azure-graph.version} + + + com.microsoft.graph + microsoft-graph-core + 3.6.5 + + + com.microsoft.kiota + microsoft-kiota-abstractions + 1.9.0 + + + com.microsoft.kiota + microsoft-kiota-authentication-azure + 1.9.0 + + + com.microsoft.kiota + microsoft-kiota-http-okHttp + 1.9.0 + + + com.microsoft.kiota + microsoft-kiota-serialization-form + 1.9.0 + + + com.microsoft.kiota + microsoft-kiota-serialization-json + 1.9.0 + + + com.microsoft.kiota + microsoft-kiota-serialization-multipart + 1.9.0 + + + com.microsoft.kiota + microsoft-kiota-serialization-text + 1.9.0 + + + io.github.std-uritemplate + std-uritemplate + 2.0.0 + + + + com.azure + azure-core + provided + + + com.azure + azure-core-http-netty + provided + + + io.netty + * + + + + + + com.azure + azure-identity + provided + + + com.azure + azure-json + provided + + + com.azure + azure-xml + provided + + + com.google.code.gson + gson + provided + + + com.squareup.okhttp3 + okhttp + provided + + + com.squareup.okio + okio + provided + + + com.squareup.okio + okio-jvm + provided + + + io.netty + netty-buffer + provided + + + io.netty + netty-codec-http + provided + + + io.netty + netty-codec-http2 + provided + + + io.netty + netty-codec-http3 + provided + + + io.netty + netty-codec-native-quic + + + + + io.netty + netty-common + provided + + + io.netty + netty-handler + provided + + + io.netty + netty-handler-proxy + provided + + + io.netty + netty-resolver + provided + + + io.netty + netty-resolver-dns + provided + + + io.netty + netty-resolver-dns-native-macos + osx-x86_64 + provided + + + io.netty + netty-resolver-dns-native-macos + osx-aarch_64 + provided + + + io.netty + netty-transport + provided + + + io.netty + netty-transport-native-unix-common + provided + + + io.projectreactor + reactor-core + provided + + + org.jspecify + jspecify + provided + + + org.reactivestreams + reactive-streams + provided + + + org.junit.jupiter + junit-jupiter + ${junit.version} + test + + + + diff --git a/plugins/tech/microsoft365/src/assembly/assembly.xml b/plugins/tech/microsoft365/src/assembly/assembly.xml new file mode 100644 index 00000000000..0108fcfb455 --- /dev/null +++ b/plugins/tech/microsoft365/src/assembly/assembly.xml @@ -0,0 +1,37 @@ + + + + hop-tech-microsoft365 + + zip + + . + + + ${project.basedir}/src/main/resources/version.xml + ${hop.plugin.libdir} + true + + + + + ${maven.multiModuleProjectDirectory}/assemblies/shared/hop-plugin-libs.xml + + diff --git a/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileName.java b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileName.java new file mode 100644 index 00000000000..39080709f83 --- /dev/null +++ b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileName.java @@ -0,0 +1,135 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.vfs.sharepoint; + +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.StringJoiner; +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.provider.AbstractFileName; +import org.apache.commons.vfs2.provider.UriParser; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * Named Microsoft 365 connection URI: {@code connectionName:///site/library/path/in/the/library}. + */ +@NullMarked +public class SharePointFileName extends AbstractFileName { + + /** Depth of a site: {@code connection:///site}. */ + public static final int SITE_DEPTH = 1; + + /** Depth of a document library (drive): {@code connection:///site/library}. */ + public static final int DRIVE_DEPTH = 2; + + /** The decoded names of the path: the site, the library, then the folders and the file. */ + private final String[] segments; + + public SharePointFileName(final String scheme, final String path, final FileType type) { + super(scheme, path, type); + this.segments = split(getPath()); + } + + /** + * URL encodes a name for a Microsoft Graph path-based address, where a space is {@code %20} and + * not a plus sign. + */ + static String encodeSegment(String name) { + return URLEncoder.encode(name, StandardCharsets.UTF_8).replace("+", "%20"); + } + + /** + * @return the name of the site or null for the root of the file system + */ + public @Nullable String getSiteName() { + return segments.length >= SITE_DEPTH ? segments[SITE_DEPTH - 1] : null; + } + + /** + * @return the name of the document library or null above the libraries + */ + public @Nullable String getDriveName() { + return segments.length >= DRIVE_DEPTH ? segments[DRIVE_DEPTH - 1] : null; + } + + /** + * The path of the item relative to the root of its document library, every segment URL encoded so + * it can be used in a Microsoft Graph path-based address. + * + * @return the encoded path without leading slash, empty for the library itself and above + */ + public String getEncodedItemPath() { + StringJoiner joiner = new StringJoiner("/"); + for (int i = DRIVE_DEPTH; i < segments.length; i++) { + joiner.add(encodeSegment(segments[i])); + } + return joiner.toString(); + } + + /** + * @return true if this name is an item (file or folder) inside a document library + */ + public boolean isItem() { + return getLevel() > DRIVE_DEPTH; + } + + /** + * @return the number of names in the path: 0 for the root, 1 for a site, 2 for a library + */ + public int getLevel() { + return segments.length; + } + + /** + * @return the base name as Microsoft Graph knows it, without the VFS escaping of the path + */ + public String getDecodedBaseName() { + return segments.length == 0 ? "" : segments[segments.length - 1]; + } + + private static String[] split(String path) { + if (path.length() <= 1) { + return new String[0]; + } + // The path keeps a literal percent sign escaped + String[] segments = path.substring(1).split("/"); + for (int i = 0; i < segments.length; i++) { + try { + segments[i] = UriParser.decode(segments[i]); + } catch (FileSystemException e) { + // Not an escape sequence, the name is used as it is + } + } + return segments; + } + + @Override + public FileName createName(String absPath, FileType type) { + return new SharePointFileName(getScheme(), absPath, type); + } + + @Override + protected void appendRootUri(StringBuilder buffer, boolean addPassword) { + buffer.append(getScheme()); + buffer.append("://"); + } +} diff --git a/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileNameParser.java b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileNameParser.java new file mode 100644 index 00000000000..7a01fc43d0f --- /dev/null +++ b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileNameParser.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.sharepoint; + +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.provider.AbstractFileNameParser; +import org.apache.commons.vfs2.provider.FileNameParser; +import org.apache.commons.vfs2.provider.UriParser; +import org.apache.commons.vfs2.provider.VfsComponentContext; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * Parses {@code scheme:///site/library/path}. The scheme is the named connection. Credentials are + * not in the URI. + */ +@NullMarked +public class SharePointFileNameParser extends AbstractFileNameParser { + private static final SharePointFileNameParser INSTANCE = new SharePointFileNameParser(); + + public static FileNameParser getInstance() { + return INSTANCE; + } + + @Override + public FileName parseUri( + @Nullable VfsComponentContext context, @Nullable FileName base, String uri) + throws FileSystemException { + StringBuilder name = new StringBuilder(); + String scheme = UriParser.extractScheme(uri, name); + if (scheme == null) { + if (base == null) { + throw new FileSystemException("vfs.provider/absolute-uri.error", uri); + } + scheme = base.getScheme(); + if (name.isEmpty() || name.charAt(0) != '/') { + name.insert(0, base.getPath().endsWith("/") ? base.getPath() : base.getPath() + "/"); + } + } + UriParser.canonicalizePath(name, 0, name.length(), this); + UriParser.fixSeparators(name); + while (!name.isEmpty() && name.charAt(0) == '/') { + name.deleteCharAt(0); + } + name.insert(0, '/'); + FileType fileType = UriParser.normalisePath(name); + String path = name.toString(); + if (path.isEmpty() || "/".equals(path)) { + path = "/"; + fileType = FileType.FOLDER; + } + return new SharePointFileName(scheme, path, fileType); + } +} diff --git a/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileObject.java b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileObject.java new file mode 100644 index 00000000000..f0c8f7009fd --- /dev/null +++ b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileObject.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.vfs.sharepoint; + +import com.microsoft.graph.models.DriveItem; +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.vfs2.FileNotFoundException; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileType; +import org.apache.commons.vfs2.NameScope; +import org.apache.commons.vfs2.provider.AbstractFileName; +import org.apache.commons.vfs2.provider.AbstractFileObject; +import org.apache.commons.vfs2.provider.UriParser; +import org.apache.commons.vfs2.util.FileObjectUtils; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * A site, a document library, or a file or folder of a document library, depending on the depth of + * its name. + */ +@NullMarked +public class SharePointFileObject extends AbstractFileObject { + + /** The id of the site, only for a site. */ + private @Nullable String siteId; + + /** The id of the document library, for a library and everything below. */ + private @Nullable String driveId; + + /** The metadata of the file or folder, only below a document library. */ + private @Nullable DriveItem item; + + protected SharePointFileObject(AbstractFileName name, SharePointFileSystem fileSystem) { + super(name, fileSystem); + } + + private SharePointFileName name() { + return (SharePointFileName) getName(); + } + + @Override + protected void doAttach() throws Exception { + // Already filled in by the listing of the parent folder + if (item != null) { + return; + } + SharePointFileName name = name(); + SharePointFileSystem fileSystem = getAbstractFileSystem(); + int depth = name.getLevel(); + if (depth == SharePointFileName.SITE_DEPTH) { + siteId = fileSystem.findSiteId(name.getSiteName()); + } else if (depth >= SharePointFileName.DRIVE_DEPTH) { + driveId = fileSystem.findDriveId(name); + if (driveId != null && name.isItem()) { + item = fileSystem.findItem(driveId, name.getEncodedItemPath()); + } + } + } + + @Override + protected void doDetach() { + siteId = null; + driveId = null; + item = null; + } + + @Override + protected FileType doGetType() { + int depth = name().getLevel(); + if (depth < SharePointFileName.SITE_DEPTH) { + return FileType.FOLDER; + } + if (depth == SharePointFileName.SITE_DEPTH) { + return siteId == null ? FileType.IMAGINARY : FileType.FOLDER; + } + if (depth == SharePointFileName.DRIVE_DEPTH) { + return driveId == null ? FileType.IMAGINARY : FileType.FOLDER; + } + return typeOf(item); + } + + private static FileType typeOf(@Nullable DriveItem item) { + if (item == null) { + return FileType.IMAGINARY; + } + // A package, such as a OneNote notebook, is browsed like a folder + return item.getFolder() != null || item.getPackage() != null ? FileType.FOLDER : FileType.FILE; + } + + @Override + protected long doGetContentSize() { + return item == null || item.getSize() == null ? 0 : item.getSize(); + } + + @Override + protected long doGetLastModifiedTime() { + return item == null || item.getLastModifiedDateTime() == null + ? 0 + : item.getLastModifiedDateTime().toInstant().toEpochMilli(); + } + + @Override + protected String @Nullable [] doListChildren() throws Exception { + // A file, or a site, library or folder that does not exist + if (doGetType() != FileType.FOLDER) { + return null; + } + SharePointFileSystem fileSystem = getAbstractFileSystem(); + int depth = name().getLevel(); + List names = new ArrayList<>(); + if (depth < SharePointFileName.SITE_DEPTH) { + names.addAll(fileSystem.listSites().keySet()); + } else if (depth == SharePointFileName.SITE_DEPTH) { + names.addAll(fileSystem.listDrives(siteId).keySet()); + } else { + for (DriveItem child : fileSystem.listChildren(driveId, folderItemId())) { + names.add(child.getName()); + } + } + // The names go through the file name parser, which decodes them + return names.stream().map(UriParser::encode).toArray(String[]::new); + } + + @Override + protected FileObject @Nullable [] doListChildrenResolved() throws Exception { + // The sites and the document libraries are resolved by name, anything that is not a folder is + // rejected by the listing of the names + if (name().getLevel() < SharePointFileName.DRIVE_DEPTH || doGetType() != FileType.FOLDER) { + return null; + } + SharePointFileSystem fileSystem = getAbstractFileSystem(); + List children = fileSystem.listChildren(driveId, folderItemId()); + List files = new ArrayList<>(children.size()); + for (DriveItem child : children) { + FileObject file = + fileSystem.resolveFile( + fileSystem + .getFileSystemManager() + .resolveName(getName(), UriParser.encode(child.getName()), NameScope.CHILD)); + // Hand over the metadata so the child does not have to fetch it again + if (FileObjectUtils.getAbstractFileObject(file) + instanceof SharePointFileObject sharePointFile) { + sharePointFile.inject(driveId, child); + } + files.add(file); + } + return files.toArray(new FileObject[0]); + } + + private void inject(String driveId, DriveItem item) throws FileSystemException { + this.driveId = driveId; + this.item = item; + injectType(typeOf(item)); + } + + /** + * @return the id of this folder to list its children, a document library is its own root folder + */ + private String folderItemId() { + return item == null ? SharePointFileSystem.ROOT_ITEM_ID : item.getId(); + } + + @Override + protected InputStream doGetInputStream() throws Exception { + if (item == null) { + throw new FileNotFoundException(getName()); + } + InputStream stream = getAbstractFileSystem().download(driveId, item.getId()); + if (stream == null) { + throw new FileNotFoundException(getName()); + } + return stream; + } + + @Override + protected OutputStream doGetOutputStream(boolean append) throws Exception { + if (append) { + throw new FileSystemException("vfs.provider/write-append-not-supported.error", getName()); + } + final String targetDriveId = requireDriveId(); + final String encodedPath = name().getEncodedItemPath(); + + // Microsoft Graph needs the size before the first byte is sent, so the content is buffered + final Path buffer = Files.createTempFile("hop-sharepoint-", ".tmp"); + return new FilterOutputStream(Files.newOutputStream(buffer)) { + private boolean closed; + + @Override + public void write(byte[] bytes, int offset, int length) throws IOException { + out.write(bytes, offset, length); + } + + @Override + public void close() throws IOException { + if (closed) { + return; + } + closed = true; + try { + super.close(); + driveId = targetDriveId; + item = getAbstractFileSystem().upload(targetDriveId, encodedPath, buffer); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException("Failed to upload " + getName(), e); + } finally { + Files.deleteIfExists(buffer); + } + } + }; + } + + @Override + protected void doCreateFolder() throws Exception { + String targetDriveId = requireDriveId(); + SharePointFileName parent = (SharePointFileName) getName().getParent(); + driveId = targetDriveId; + item = + getAbstractFileSystem() + .createFolder( + targetDriveId, + requireItemId(targetDriveId, parent), + name().getDecodedBaseName(), + name().getEncodedItemPath()); + } + + @Override + protected void doDelete() throws Exception { + String targetDriveId = requireDriveId(); + if (item == null) { + throw new FileNotFoundException(getName()); + } + getAbstractFileSystem().delete(targetDriveId, item.getId()); + item = null; + } + + /** An item can only be moved inside its document library, anything else is a copy and delete. */ + @Override + public boolean canRenameTo(FileObject newFile) { + if (!super.canRenameTo(newFile) || !(newFile.getName() instanceof SharePointFileName target)) { + return false; + } + SharePointFileName source = name(); + return source.isItem() + && target.isItem() + && source.getSiteName().equalsIgnoreCase(target.getSiteName()) + && source.getDriveName().equalsIgnoreCase(target.getDriveName()); + } + + @Override + protected void doRename(FileObject newFile) throws Exception { + if (item == null) { + throw new FileNotFoundException(getName()); + } + String targetDriveId = requireDriveId(); + SharePointFileName target = (SharePointFileName) newFile.getName(); + getAbstractFileSystem() + .move( + targetDriveId, + item.getId(), + requireItemId(targetDriveId, (SharePointFileName) target.getParent()), + target.getDecodedBaseName()); + item = null; + } + + /** + * @return the id of the document library of this file or folder + * @throws FileSystemException for the root, a site or a library, which can only be read, or when + * the site or the library does not exist + */ + private String requireDriveId() throws FileSystemException { + if (!name().isItem()) { + throw new FileSystemException( + "Sites and document libraries are read-only, \"" + getName() + "\" can not be modified"); + } + String id = driveId != null ? driveId : getAbstractFileSystem().findDriveId(name()); + if (id == null) { + throw new FileSystemException( + "The site or document library of \"" + getName() + "\" does not exist"); + } + return id; + } + + private String requireItemId(String driveId, SharePointFileName folder) + throws FileSystemException { + DriveItem folderItem = getAbstractFileSystem().findItem(driveId, folder.getEncodedItemPath()); + if (folderItem == null) { + throw new FileNotFoundException(folder); + } + return folderItem.getId(); + } +} diff --git a/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileProvider.java b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileProvider.java new file mode 100644 index 00000000000..f9266e1144f --- /dev/null +++ b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileProvider.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.vfs.sharepoint; + +import com.azure.core.credential.TokenCredential; +import com.microsoft.graph.serviceclient.GraphServiceClient; +import java.util.Collection; +import org.apache.commons.vfs2.Capability; +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileSystem; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemOptions; +import org.apache.commons.vfs2.provider.AbstractOriginatingFileProvider; +import org.apache.hop.core.logging.LogChannel; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.vfs.sharepoint.metadatatype.SharePointConnection; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +@NullMarked +public class SharePointFileProvider extends AbstractOriginatingFileProvider { + + /** The .default scope is required by app-only authentication. */ + private static final String[] SCOPES = {"https://graph.microsoft.com/.default"}; + + private final IVariables variables; + private final SharePointConnection connection; + + public SharePointFileProvider(IVariables variables, SharePointConnection connection) { + this.variables = variables; + this.connection = connection; + setFileNameParser(SharePointFileNameParser.getInstance()); + } + + @Override + public Collection getCapabilities() { + return SharePointFileSystem.CAPABILITIES; + } + + @Override + protected FileSystem doCreateFileSystem( + FileName name, @Nullable FileSystemOptions fileSystemOptions) throws FileSystemException { + try { + LogChannel.GENERAL.logDetailed( + "Initialize Microsoft Graph client for " + connection.getName()); + TokenCredential credential = connection.credentials(LogChannel.GENERAL, variables); + GraphServiceClient client = new GraphServiceClient(credential, SCOPES); + return new SharePointFileSystem( + name, client, variables.resolve(connection.getHostname()), fileSystemOptions); + } catch (Exception e) { + throw new FileSystemException( + "Unable to connect to Microsoft Graph with connection " + connection.getName(), e); + } + } +} diff --git a/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileSystem.java b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileSystem.java new file mode 100644 index 00000000000..21108b7fa42 --- /dev/null +++ b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointFileSystem.java @@ -0,0 +1,503 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.vfs.sharepoint; + +import com.microsoft.graph.core.models.UploadResult; +import com.microsoft.graph.core.tasks.LargeFileUploadTask; +import com.microsoft.graph.drives.item.items.item.DriveItemItemRequestBuilder; +import com.microsoft.graph.drives.item.items.item.children.ChildrenRequestBuilder; +import com.microsoft.graph.drives.item.items.item.createuploadsession.CreateUploadSessionPostRequestBody; +import com.microsoft.graph.models.BaseCollectionPaginationCountResponse; +import com.microsoft.graph.models.DriveCollectionResponse; +import com.microsoft.graph.models.DriveItem; +import com.microsoft.graph.models.DriveItemCollectionResponse; +import com.microsoft.graph.models.DriveItemUploadableProperties; +import com.microsoft.graph.models.Folder; +import com.microsoft.graph.models.ItemReference; +import com.microsoft.graph.models.Site; +import com.microsoft.graph.models.SiteCollectionResponse; +import com.microsoft.graph.models.UploadSession; +import com.microsoft.graph.models.odataerrors.ODataError; +import com.microsoft.graph.serviceclient.GraphServiceClient; +import com.microsoft.graph.sites.item.drives.DrivesRequestBuilder; +import com.microsoft.kiota.ApiException; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Consumer; +import java.util.function.Function; +import org.apache.commons.vfs2.Capability; +import org.apache.commons.vfs2.FileName; +import org.apache.commons.vfs2.FileObject; +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileSystemOptions; +import org.apache.commons.vfs2.provider.AbstractFileName; +import org.apache.commons.vfs2.provider.AbstractFileSystem; +import org.apache.hop.core.logging.LogChannel; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +/** + * One named Microsoft 365 connection. The sites are the folders of the root, the document libraries + * (drives) of a site are the folders of that site. + */ +@NullMarked +public class SharePointFileSystem extends AbstractFileSystem { + + public static final List CAPABILITIES = + List.of( + Capability.CREATE, + Capability.DELETE, + Capability.RENAME, + Capability.GET_TYPE, + Capability.GET_LAST_MODIFIED, + Capability.LIST_CHILDREN, + Capability.READ_CONTENT, + Capability.URI, + Capability.WRITE_CONTENT); + + /** Above this size the content is sent in slices through an upload session. */ + static final long SIMPLE_UPLOAD_LIMIT = 4L * 1024 * 1024; + + /** The id that addresses the root folder of a document library, and the root site of a tenant. */ + static final String ROOT_ITEM_ID = "root"; + + /** A site created by a Microsoft Teams team may live under /teams. */ + private static final List MANAGED_PATHS = List.of("sites", "teams"); + + /** The only properties of a file or folder that are used, the others are not requested. */ + private static final String[] ITEM_PROPERTIES = { + "id", "name", "size", "lastModifiedDateTime", "file", "folder", "package" + }; + + private static final String CONFLICT_BEHAVIOR = "@microsoft.graph.conflictBehavior"; + private static final int HTTP_NOT_FOUND = 404; + private static final int HTTP_CONFLICT = 409; + + private final GraphServiceClient client; + + /** Site name to site id, whatever the case of the name, null until the sites are listed. */ + private volatile @Nullable Map siteIds; + + /** The SharePoint host of the tenant, null until it is needed. */ + private volatile @Nullable String hostname; + + /** False once Microsoft Graph refused to list the sites. */ + private volatile boolean sitesListable = true; + + /** Site id to (drive name to drive id, whatever the case of the name). */ + private final Map> driveIds = new ConcurrentHashMap<>(); + + public SharePointFileSystem( + FileName rootName, + GraphServiceClient client, + @Nullable String hostname, + @Nullable FileSystemOptions fileSystemOptions) { + super(rootName, null, fileSystemOptions); + this.client = client; + this.hostname = hostname == null || hostname.isBlank() ? null : hostname.trim(); + } + + @Override + protected void addCapabilities(Collection capabilities) { + capabilities.addAll(CAPABILITIES); + } + + @Override + protected FileObject createFile(AbstractFileName name) { + return new SharePointFileObject(name, this); + } + + /** + * Lists the sites of the tenant and refreshes the cache. + * + * @return the site ids by site name + */ + Map listSites() throws FileSystemException { + Map sites = newNameMap(); + try { + addSites(sites, client.sites().get()); + } catch (ApiException e) { + throw new FileSystemException( + "Microsoft Graph refused to list the SharePoint sites: " + describe(e), e); + } + if (sites.isEmpty()) { + // Depending on the tenant the plain listing is empty and only the search returns the sites + try { + String url = client.getRequestAdapter().getBaseUrl() + "/sites?search=*"; + addSites(sites, client.sites().withUrl(url).get()); + } catch (ApiException e) { + LogChannel.GENERAL.logBasic( + "Microsoft Graph refused to search the SharePoint sites: " + describe(e)); + } + } + if (sites.isEmpty()) { + LogChannel.GENERAL.logBasic( + "Microsoft Graph returned no SharePoint site. Listing the sites needs the application" + + " permission Sites.Read.All with admin consent. With Sites.Selected the sites" + + " can not be listed, use the full path of a site the application was granted."); + } + siteIds = sites; + driveIds.clear(); + return sites; + } + + /** + * @return the HTTP status, the error code and the message of Microsoft Graph + */ + static String describe(ApiException e) { + StringBuilder text = new StringBuilder("HTTP ").append(e.getResponseStatusCode()); + if (e instanceof ODataError error && error.getError() != null) { + text.append(' ').append(error.getError().getCode()); + text.append(": ").append(error.getError().getMessage()); + } else if (e.getMessage() != null) { + text.append(": ").append(e.getMessage()); + } + return text.toString(); + } + + private void addSites(Map sites, @Nullable SiteCollectionResponse firstPage) { + forEachPaged( + firstPage, + SiteCollectionResponse::getValue, + nextLink -> client.sites().withUrl(nextLink).get(), + site -> addSite(sites, site)); + } + + /** Two sites can have the same name, only the first one is kept. */ + private static void addSite(Map sites, Site site) { + String name = site.getName() != null ? site.getName() : site.getDisplayName(); + if (name == null || site.getId() == null) { + return; + } + String namesake = sites.putIfAbsent(name, site.getId()); + if (namesake != null && !namesake.equals(site.getId())) { + LogChannel.GENERAL.logBasic( + "The SharePoint site " + + site.getWebUrl() + + " is ignored: another site is already named '" + + name + + "'"); + } + } + + /** + * Reads a site by its address, {@code /sites/name} of the SharePoint host of the tenant. This + * also works when the application can not list the sites (Sites.Selected). + * + * @return the id of the site or null if it does not exist or can not be read + */ + private @Nullable String findSiteIdByPath(String siteName) { + String host; + try { + host = hostname(); + } catch (ApiException e) { + LogChannel.GENERAL.logBasic( + "The SharePoint host of the tenant can not be read, set it on the connection: " + + describe(e)); + return null; + } + if (host == null) { + return null; + } + String encodedSiteName = SharePointFileName.encodeSegment(siteName); + for (String managedPath : MANAGED_PATHS) { + // The address of the site relative to the host, such as contoso.sharepoint.com:/sites/Finance + String address = host + ":/" + managedPath + "/"; + String url = client.getRequestAdapter().getBaseUrl() + "/sites/" + address + encodedSiteName; + try { + Site site = client.sites().bySiteId(ROOT_ITEM_ID).withUrl(url).get(); + if (site != null && site.getId() != null) { + return site.getId(); + } + } catch (ApiException e) { + LogChannel.GENERAL.logBasic( + "The SharePoint site " + + host + + "/" + + managedPath + + "/" + + siteName + + " can not be read: " + + describe(e)); + } + } + return null; + } + + /** + * Lists the sites, an application that is only granted some sites (Sites.Selected) can not list + * them and reaches its sites by their address. + */ + private Map listSitesIfAllowed() { + try { + return listSites(); + } catch (FileSystemException e) { + LogChannel.GENERAL.logDetailed(e.getMessage()); + Map sites = newNameMap(); + sitesListable = false; + siteIds = sites; + return sites; + } + } + + /** + * @return the SharePoint host of the tenant, such as contoso.sharepoint.com, or null if the root + * site can not be read + */ + private @Nullable String hostname() { + if (hostname == null) { + Site root = client.sites().bySiteId(ROOT_ITEM_ID).get(); + if (root != null && root.getSiteCollection() != null) { + hostname = root.getSiteCollection().getHostname(); + } + if (hostname == null && root != null && root.getWebUrl() != null) { + hostname = URI.create(root.getWebUrl()).getHost(); + } + } + return hostname; + } + + /** + * @return the id of the site or null if there is no site with this name + */ + @Nullable String findSiteId(String siteName) throws FileSystemException { + Map sites = siteIds; + boolean cached = sites != null; + if (!cached) { + sites = listSitesIfAllowed(); + } + String siteId = sites.get(siteName); + // The site may have been created since the sites were listed + if (siteId == null && cached && sitesListable) { + sites = listSitesIfAllowed(); + siteId = sites.get(siteName); + } + // The site may be readable without being listed + if (siteId == null) { + siteId = findSiteIdByPath(siteName); + if (siteId != null) { + sites.put(siteName, siteId); + } + } + return siteId; + } + + /** + * Lists the document libraries of a site and refreshes the cache. + * + * @return the drive ids by drive name + */ + Map listDrives(String siteId) { + Map drives = newNameMap(); + DrivesRequestBuilder request = client.sites().bySiteId(siteId).drives(); + forEachPaged( + request.get(), + DriveCollectionResponse::getValue, + nextLink -> request.withUrl(nextLink).get(), + drive -> { + if (drive.getName() != null && drive.getId() != null) { + drives.putIfAbsent(drive.getName(), drive.getId()); + } + }); + driveIds.put(siteId, drives); + return drives; + } + + /** + * @return the id of the document library or null if the site or the library does not exist + */ + @Nullable String findDriveId(SharePointFileName name) throws FileSystemException { + String siteId = findSiteId(name.getSiteName()); + if (siteId == null) { + return null; + } + Map drives = driveIds.get(siteId); + boolean cached = drives != null; + if (!cached) { + drives = listDrives(siteId); + } + String driveId = drives.get(name.getDriveName()); + // The library may have been created since the libraries were listed + if (driveId == null && cached) { + driveId = listDrives(siteId).get(name.getDriveName()); + } + return driveId; + } + + /** + * Reads the metadata of an item addressed by its path in the document library. + * + * @param encodedPath the URL encoded path, empty for the root folder of the library + * @return the item or null if it does not exist + */ + @Nullable DriveItem findItem(String driveId, String encodedPath) { + try { + return item(driveId, ROOT_ITEM_ID).withUrl(itemUrl(driveId, encodedPath)).get(); + } catch (ApiException e) { + if (e.getResponseStatusCode() == HTTP_NOT_FOUND) { + return null; + } + throw e; + } + } + + List listChildren(String driveId, String itemId) { + List children = new ArrayList<>(); + ChildrenRequestBuilder request = item(driveId, itemId).children(); + forEachPaged( + // A folder can hold thousands of items, the unused properties are left out of the answer + request.get(configuration -> configuration.queryParameters.select = ITEM_PROPERTIES), + DriveItemCollectionResponse::getValue, + nextLink -> request.withUrl(nextLink).get(), + children::add); + return children; + } + + /** + * Hands every element of a Microsoft Graph collection to a consumer, following the links to the + * next pages. + * + * @param firstPage the answer to the request of the collection + * @param values reads the elements of a page + * @param nextPage requests a page with the link given by the previous one + */ + private static

void forEachPaged( + @Nullable P firstPage, + Function> values, + Function nextPage, + Consumer consumer) { + P page = firstPage; + while (page != null) { + List pageValues = values.apply(page); + if (pageValues != null) { + pageValues.forEach(consumer); + } + String nextLink = page.getOdataNextLink(); + page = nextLink == null ? null : nextPage.apply(nextLink); + } + } + + /** A map keyed by the name of a site or of a document library, whatever the case of the name. */ + private static Map newNameMap() { + return Collections.synchronizedMap(new TreeMap<>(String.CASE_INSENSITIVE_ORDER)); + } + + @Nullable InputStream download(String driveId, String itemId) { + return item(driveId, itemId).content().get(); + } + + /** + * Creates or replaces a file with the content of a local file. + * + * @param encodedPath the URL encoded path of the file in the document library + * @return the uploaded item + */ + @Nullable DriveItem upload(String driveId, String encodedPath, Path content) throws Exception { + long size = Files.size(content); + String url = itemUrl(driveId, encodedPath); + if (size <= SIMPLE_UPLOAD_LIMIT) { + // An in-memory stream has a known length and can be sent again when the request is retried + InputStream stream = new ByteArrayInputStream(Files.readAllBytes(content)); + return item(driveId, ROOT_ITEM_ID).content().withUrl(url + ":/content").put(stream); + } + try (InputStream stream = Files.newInputStream(content)) { + DriveItemUploadableProperties properties = new DriveItemUploadableProperties(); + properties.getAdditionalData().put(CONFLICT_BEHAVIOR, "replace"); + CreateUploadSessionPostRequestBody body = new CreateUploadSessionPostRequestBody(); + body.setItem(properties); + UploadSession session = + item(driveId, ROOT_ITEM_ID) + .createUploadSession() + .withUrl(url + ":/createUploadSession") + .post(body); + + LargeFileUploadTask task = + new LargeFileUploadTask<>( + client.getRequestAdapter(), + session, + stream, + size, + DriveItem::createFromDiscriminatorValue); + UploadResult result = task.upload(); + if (!result.isUploadSuccessful()) { + throw new IOException("The upload session did not complete"); + } + // The session does not always return the item + return result.itemResponse != null ? result.itemResponse : findItem(driveId, encodedPath); + } + } + + /** + * Creates a folder, a folder created by someone else in the meantime is not an error. + * + * @param encodedPath the URL encoded path of the folder in the document library + * @return the created or existing folder + */ + DriveItem createFolder( + String driveId, String parentItemId, String folderName, String encodedPath) { + DriveItem folder = new DriveItem(); + folder.setName(folderName); + folder.setFolder(new Folder()); + folder.getAdditionalData().put(CONFLICT_BEHAVIOR, "fail"); + try { + return item(driveId, parentItemId).children().post(folder); + } catch (ApiException e) { + DriveItem existing = + e.getResponseStatusCode() == HTTP_CONFLICT ? findItem(driveId, encodedPath) : null; + if (existing == null || existing.getFolder() == null) { + throw e; + } + return existing; + } + } + + void delete(String driveId, String itemId) { + item(driveId, itemId).delete(); + } + + /** Renames an item and moves it to another folder of the same document library. */ + DriveItem move(String driveId, String itemId, String newParentItemId, String newName) { + ItemReference parent = new ItemReference(); + parent.setId(newParentItemId); + DriveItem update = new DriveItem(); + update.setName(newName); + update.setParentReference(parent); + return item(driveId, itemId).patch(update); + } + + private DriveItemItemRequestBuilder item(String driveId, String itemId) { + return client.drives().byDriveId(driveId).items().byDriveItemId(itemId); + } + + private String itemUrl(String driveId, String encodedPath) { + String url = client.getRequestAdapter().getBaseUrl() + "/drives/" + driveId + "/root"; + return encodedPath.isEmpty() ? url : url + ":/" + encodedPath; + } +} diff --git a/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointVfsPlugin.java b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointVfsPlugin.java new file mode 100644 index 00000000000..6b5b699d6dd --- /dev/null +++ b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/SharePointVfsPlugin.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hop.vfs.sharepoint; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.vfs2.provider.FileProvider; +import org.apache.hop.core.logging.LogChannel; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.plugin.IVfs; +import org.apache.hop.core.vfs.plugin.VfsPlugin; +import org.apache.hop.metadata.api.IHopMetadataProvider; +import org.apache.hop.metadata.util.HopMetadataUtil; +import org.apache.hop.vfs.sharepoint.metadatatype.SharePointConnection; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +@NullMarked +@VfsPlugin( + type = "sharepoint", + typeDescription = "Microsoft SharePoint VFS", + classLoaderGroup = "vfs-graph") +public class SharePointVfsPlugin implements IVfs { + + @Override + public String[] getUrlSchemes() { + // Named connections only. Credentials stay on the metadata, never in a URI. + return new String[] {}; + } + + @Override + public @Nullable FileProvider getProvider() { + return null; + } + + @Override + public Map getProviders(IVariables variables) { + return getProviders(variables, null); + } + + @Override + public Map getProviders( + IVariables variables, @Nullable IHopMetadataProvider executionMetadata) { + Map providers = new HashMap<>(); + try { + IHopMetadataProvider metadataProvider = + executionMetadata != null + ? executionMetadata + : HopMetadataUtil.getStandardHopMetadataProvider(variables); + List connections = + metadataProvider.getSerializer(SharePointConnection.class).loadAll(); + for (SharePointConnection connection : connections) { + String name = connection.getName(); + if (StringUtils.isEmpty(name)) { + continue; + } + providers.put(name, new SharePointFileProvider(variables, connection)); + } + } catch (Exception e) { + LogChannel.GENERAL.logError("Unable to load the SharePoint VFS providers", e); + } + return providers; + } +} diff --git a/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnection.java b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnection.java new file mode 100644 index 00000000000..ad441daadc8 --- /dev/null +++ b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnection.java @@ -0,0 +1,224 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.sharepoint.metadatatype; + +import com.azure.core.credential.TokenCredential; +import com.azure.identity.ClientCertificateCredentialBuilder; +import com.azure.identity.ClientSecretCredentialBuilder; +import com.azure.identity.DefaultAzureCredentialBuilder; +import java.io.ByteArrayInputStream; +import java.nio.charset.StandardCharsets; +import lombok.Getter; +import lombok.Setter; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.gui.plugin.GuiElementType; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.core.gui.plugin.GuiWidgetElement; +import org.apache.hop.core.logging.ILogChannel; +import org.apache.hop.core.variables.IVariables; +import org.apache.hop.core.vfs.IVfsBrowseLocation; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.metadata.api.HopMetadata; +import org.apache.hop.metadata.api.HopMetadataBase; +import org.apache.hop.metadata.api.HopMetadataCategory; +import org.apache.hop.metadata.api.HopMetadataProperty; +import org.apache.hop.metadata.api.HopMetadataPropertyType; +import org.apache.hop.metadata.api.IEnumHasCode; +import org.apache.hop.metadata.api.IEnumHasCodeAndDescription; +import org.apache.hop.metadata.api.IHopMetadata; +import org.jspecify.annotations.NullMarked; +import org.jspecify.annotations.Nullable; + +@Getter +@Setter +@NullMarked +@GuiPlugin(classLoaderGroup = "vfs-graph") +@HopMetadata( + key = "sharepoint", + name = "i18n::SharePointConnection.Name", + description = "i18n::SharePointConnection.Description", + image = "sharepoint.svg", + category = HopMetadataCategory.FILE_STORAGE, + documentationUrl = "/metadata-types/sharepoint-connection.html", + hopMetadataPropertyType = HopMetadataPropertyType.VFS_SHAREPOINT_CONNECTION, + classLoaderGroup = "vfs-graph") +public class SharePointConnection extends HopMetadataBase + implements IHopMetadata, IVfsBrowseLocation { + private static final Class PKG = SharePointConnection.class; + + public static final String WIDGET_AUTHENTICATION_TYPE = "sharepoint-authentication-type"; + public static final String WIDGET_CLIENT_ID = "sharepoint-client-id"; + public static final String WIDGET_CLIENT_SECRET = "sharepoint-client-secret"; + public static final String WIDGET_CERTIFICATE = "sharepoint-certificate"; + + public enum AuthenticationType implements IEnumHasCodeAndDescription { + + /** Managed Identity. */ + MANAGED_IDENTITY("SharePointAuthenticationType.ManagedIdentity"), + /** Client Secret authentication. */ + SECRET("SharePointAuthenticationType.Secret"), + /** Client Certificate authentication. */ + CERTIFICATE("SharePointAuthenticationType.Certificate"); + + /** The authentification type description. */ + @Getter private final String description; + + AuthenticationType(String description) { + this.description = BaseMessages.getString(PKG, description); + } + + @Override + public String getCode() { + return name(); + } + + /** + * Return the authentification type for a certain code + * + * @param code the code to look for + * @return the authentification type or MANAGED_IDENTITY if nothing matches. + */ + public static AuthenticationType lookupCode(String code) { + return IEnumHasCode.lookupCode(AuthenticationType.class, code, MANAGED_IDENTITY); + } + + /** + * Return the authentification type for a certain description + * + * @param description the description to look for + * @return the authentification type or MANAGED_IDENTITY if nothing matches. + */ + public static AuthenticationType lookupDescription(String description) { + return IEnumHasCodeAndDescription.lookupDescription( + AuthenticationType.class, description, MANAGED_IDENTITY); + } + } + + public SharePointConnection() { + // Default to client secret authentication + this.authenticationType = AuthenticationType.SECRET; + } + + @GuiWidgetElement( + id = "sharepoint-description", + order = "0100", + parentId = SharePointConnectionEditor.GUI_PLUGIN_ELEMENT_PARENT_ID, + type = GuiElementType.MULTI_LINE_TEXT, + multiLineTextHeight = 4, + variables = false, + label = "i18n::SharePointConnectionEditor.Description.Label") + @HopMetadataProperty + private @Nullable String description; + + @GuiWidgetElement( + id = WIDGET_AUTHENTICATION_TYPE, + order = "0200", + parentId = SharePointConnectionEditor.GUI_PLUGIN_ELEMENT_PARENT_ID, + type = GuiElementType.COMBO, + variables = false, + label = "i18n::SharePointConnectionEditor.AuthenticationType.Label") + @HopMetadataProperty(storeWithCode = true) + private AuthenticationType authenticationType; + + @GuiWidgetElement( + id = "sharepoint-tenant-id", + order = "0300", + parentId = SharePointConnectionEditor.GUI_PLUGIN_ELEMENT_PARENT_ID, + type = GuiElementType.TEXT, + label = "i18n::SharePointConnectionEditor.TenantId.Label", + toolTip = "i18n::SharePointConnectionEditor.TenantId.Tooltip") + @HopMetadataProperty + private @Nullable String tenantId; + + /** + * The SharePoint host of the tenant, such as contoso.sharepoint.com. Needed when the application + * can not read the root site of the tenant to find it. + */ + @GuiWidgetElement( + id = "sharepoint-hostname", + order = "0400", + parentId = SharePointConnectionEditor.GUI_PLUGIN_ELEMENT_PARENT_ID, + type = GuiElementType.TEXT, + label = "i18n::SharePointConnectionEditor.Hostname.Label", + toolTip = "i18n::SharePointConnectionEditor.Hostname.Tooltip") + @HopMetadataProperty + private @Nullable String hostname; + + @GuiWidgetElement( + id = WIDGET_CLIENT_ID, + order = "0500", + parentId = SharePointConnectionEditor.GUI_PLUGIN_ELEMENT_PARENT_ID, + type = GuiElementType.TEXT, + label = "i18n::SharePointConnectionEditor.ClientId.Label", + toolTip = "i18n::SharePointConnectionEditor.ClientId.Tooltip") + @HopMetadataProperty + private @Nullable String clientId; + + @GuiWidgetElement( + id = WIDGET_CLIENT_SECRET, + order = "0600", + parentId = SharePointConnectionEditor.GUI_PLUGIN_ELEMENT_PARENT_ID, + type = GuiElementType.TEXT, + password = true, + label = "i18n::SharePointConnectionEditor.ClientSecret.Label", + toolTip = "i18n::SharePointConnectionEditor.ClientSecret.Tooltip") + @HopMetadataProperty(password = true) + private @Nullable String clientSecret; + + /** Certificate with its private key in PEM format */ + @GuiWidgetElement( + id = WIDGET_CERTIFICATE, + order = "0700", + parentId = SharePointConnectionEditor.GUI_PLUGIN_ELEMENT_PARENT_ID, + type = GuiElementType.MULTI_LINE_TEXT, + multiLineTextHeight = 12, + label = "i18n::SharePointConnectionEditor.PEMCertificate.Label", + toolTip = "i18n::SharePointConnectionEditor.PEMCertificate.Tooltip") + @HopMetadataProperty(password = true) + private @Nullable String certificate; + + public TokenCredential credentials(ILogChannel log, IVariables variables) throws HopException { + + String resolvedTenantId = variables.resolve(getTenantId()); + + return switch (authenticationType) { + case SECRET -> { + log.logBasic("SharePoint authentication is using Client Secret Credentials"); + yield new ClientSecretCredentialBuilder() + .tenantId(resolvedTenantId) + .clientId(variables.resolve(getClientId())) + .clientSecret(variables.resolve(getClientSecret())) + .build(); + } + case CERTIFICATE -> { + log.logBasic("SharePoint authentication is using Client Certificate Credentials"); + String resolvedCertificate = variables.resolve(getCertificate()); + yield new ClientCertificateCredentialBuilder() + .tenantId(resolvedTenantId) + .clientId(variables.resolve(getClientId())) + .pemCertificate( + new ByteArrayInputStream(resolvedCertificate.getBytes(StandardCharsets.UTF_8))) + .build(); + } + case MANAGED_IDENTITY -> { + log.logBasic("SharePoint authentication is using Default Azure Credentials"); + yield new DefaultAzureCredentialBuilder().tenantId(resolvedTenantId).build(); + } + }; + } +} diff --git a/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnectionEditor.java b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnectionEditor.java new file mode 100644 index 00000000000..d59e28c2e7d --- /dev/null +++ b/plugins/tech/microsoft365/src/main/java/org/apache/hop/vfs/sharepoint/metadatatype/SharePointConnectionEditor.java @@ -0,0 +1,234 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hop.vfs.sharepoint.metadatatype; + +import com.azure.core.credential.TokenCredential; +import com.azure.core.credential.TokenRequestContext; +import java.util.HashSet; +import java.util.Set; +import org.apache.hop.core.Const; +import org.apache.hop.core.exception.HopException; +import org.apache.hop.core.gui.plugin.GuiPlugin; +import org.apache.hop.core.logging.LogChannel; +import org.apache.hop.core.vfs.HopVfs; +import org.apache.hop.i18n.BaseMessages; +import org.apache.hop.ui.core.PropsUi; +import org.apache.hop.ui.core.dialog.ErrorDialog; +import org.apache.hop.ui.core.dialog.MessageBox; +import org.apache.hop.ui.core.gui.GuiCompositeWidgets; +import org.apache.hop.ui.core.gui.GuiCompositeWidgetsAdapter; +import org.apache.hop.ui.core.metadata.MetadataEditor; +import org.apache.hop.ui.core.metadata.MetadataManager; +import org.apache.hop.ui.core.widget.ComboVar; +import org.apache.hop.ui.core.widget.NamingSchemeTypes; +import org.apache.hop.ui.core.widget.TextVar; +import org.apache.hop.ui.hopgui.HopGui; +import org.apache.hop.ui.hopgui.vfs.explorer.VfsFileExplorerViews; +import org.apache.hop.vfs.sharepoint.metadatatype.SharePointConnection.AuthenticationType; +import org.eclipse.swt.SWT; +import org.eclipse.swt.layout.FormAttachment; +import org.eclipse.swt.layout.FormData; +import org.eclipse.swt.layout.FormLayout; +import org.eclipse.swt.widgets.Button; +import org.eclipse.swt.widgets.Combo; +import org.eclipse.swt.widgets.Composite; +import org.eclipse.swt.widgets.Control; +import org.eclipse.swt.widgets.Label; +import org.jspecify.annotations.NullMarked; + +/** + * The name is the only widget created here, the other options are the fields of {@link + * SharePointConnection} annotated with {@code @GuiWidgetElement}. + */ +@NullMarked +@GuiPlugin( + description = "Editor for SharePoint connection metadata", + classLoaderGroup = "vfs-graph") +public class SharePointConnectionEditor extends MetadataEditor { + + private static final Class PKG = SharePointConnectionEditor.class; + + public static final String GUI_PLUGIN_ELEMENT_PARENT_ID = + "SharePointConnectionEditor-GuiWidgetsParent"; + + private TextVar wName; + private Composite wWidgetsComposite; + private GuiCompositeWidgets guiCompositeWidgets; + + public SharePointConnectionEditor( + HopGui hopGui, MetadataManager manager, SharePointConnection metadata) { + super(hopGui, manager, metadata); + } + + @Override + public void createControl(Composite parent) { + int middle = PropsUi.getInstance().getMiddlePct(); + int margin = PropsUi.getMargin() + 2; + + Label wIcon = new Label(parent, SWT.RIGHT); + wIcon.setImage(getImage()); + FormData fdIcon = new FormData(); + fdIcon.top = new FormAttachment(0, 0); + fdIcon.right = new FormAttachment(100, 0); + wIcon.setLayoutData(fdIcon); + PropsUi.setLook(wIcon); + + Label wlName = new Label(parent, SWT.RIGHT); + PropsUi.setLook(wlName); + wlName.setText(BaseMessages.getString(PKG, "SharePointConnectionEditor.Name.Label")); + FormData fdlName = new FormData(); + fdlName.top = new FormAttachment(0, margin); + fdlName.left = new FormAttachment(0, 0); + fdlName.right = new FormAttachment(middle, -margin); + wlName.setLayoutData(fdlName); + wName = + new TextVar(hopGui.getVariables(), parent, SWT.SINGLE | SWT.LEFT | SWT.BORDER) + .asNameField(NamingSchemeTypes.HOP_METADATA); + PropsUi.setLook(wName); + FormData fdName = new FormData(); + fdName.top = new FormAttachment(wlName, 0, SWT.CENTER); + fdName.left = new FormAttachment(middle, 0); + fdName.right = new FormAttachment(wIcon, -margin); + wName.setLayoutData(fdName); + + wWidgetsComposite = new Composite(parent, SWT.NONE); + PropsUi.setLook(wWidgetsComposite); + wWidgetsComposite.setLayout(new FormLayout()); + FormData fdWidgetsComposite = new FormData(); + fdWidgetsComposite.top = new FormAttachment(wName, margin); + fdWidgetsComposite.left = new FormAttachment(0, 0); + fdWidgetsComposite.right = new FormAttachment(100, 0); + fdWidgetsComposite.bottom = new FormAttachment(100, 0); + wWidgetsComposite.setLayoutData(fdWidgetsComposite); + + guiCompositeWidgets = new GuiCompositeWidgets(manager.getVariables()); + guiCompositeWidgets.createCompositeWidgets( + metadata, null, wWidgetsComposite, GUI_PLUGIN_ELEMENT_PARENT_ID, wName); + guiCompositeWidgets.setWidgetsListener( + new GuiCompositeWidgetsAdapter() { + @Override + public void widgetModified( + GuiCompositeWidgets compositeWidgets, Control changedWidget, String widgetId) { + setChanged(); + if (SharePointConnection.WIDGET_AUTHENTICATION_TYPE.equals(widgetId)) { + hideUnusedCredentials(); + } + } + }); + + setWidgetsContent(); + resetChanged(); + wName.addModifyListener(e -> setChanged()); + } + + /** The connection is a VFS scheme, the file systems are registered again to pick up a change. */ + @Override + public void save() throws HopException { + super.save(); + HopVfs.refresh(hopGui.getVariables()); + } + + @Override + public Button[] createButtonsForButtonBar(final Composite parent) { + Button wTest = new Button(parent, SWT.PUSH); + wTest.setText(BaseMessages.getString(PKG, "System.Button.Test")); + wTest.addListener(SWT.Selection, e -> testConnection()); + + Button wExplore = VfsFileExplorerViews.exploreButton(parent, this)[0]; + + return new Button[] {wTest, wExplore}; + } + + @Override + public void setWidgetsContent() { + wName.setText(Const.NVL(getMetadata().getName(), "")); + guiCompositeWidgets.setWidgetsContents( + metadata, wWidgetsComposite, GUI_PLUGIN_ELEMENT_PARENT_ID); + hideUnusedCredentials(); + } + + @Override + public void getWidgetsContent(SharePointConnection meta) { + meta.setName(wName.getText()); + guiCompositeWidgets.getWidgetsContents(meta, GUI_PLUGIN_ELEMENT_PARENT_ID); + } + + @Override + public boolean setFocus() { + if (wName == null || wName.isDisposed()) { + return false; + } + return wName.setFocus(); + } + + /** Only shows the credentials the selected authentication type uses. */ + private void hideUnusedCredentials() { + Set hidden = new HashSet<>(); + switch (selectedAuthenticationType()) { + case SECRET -> hidden.add(SharePointConnection.WIDGET_CERTIFICATE); + case CERTIFICATE -> hidden.add(SharePointConnection.WIDGET_CLIENT_SECRET); + case MANAGED_IDENTITY -> { + hidden.add(SharePointConnection.WIDGET_CLIENT_ID); + hidden.add(SharePointConnection.WIDGET_CLIENT_SECRET); + hidden.add(SharePointConnection.WIDGET_CERTIFICATE); + } + } + guiCompositeWidgets.setWidgetsHidden(metadata, hidden); + } + + private AuthenticationType selectedAuthenticationType() { + Control control = + guiCompositeWidgets.getWidgetsMap().get(SharePointConnection.WIDGET_AUTHENTICATION_TYPE); + String description = null; + if (control instanceof Combo combo) { + description = combo.getText(); + } else if (control instanceof ComboVar comboVar) { + description = comboVar.getText(); + } + // Nothing selected yet, the connection is the better answer + return description == null || description.isBlank() + ? getMetadata().getAuthenticationType() + : AuthenticationType.lookupDescription(description); + } + + private void testConnection() { + try { + SharePointConnection meta = new SharePointConnection(); + this.getWidgetsContent(meta); + + TokenCredential credential = meta.credentials(LogChannel.UI, hopGui.getVariables()); + + // Test credential + credential + .getToken(new TokenRequestContext().addScopes("https://graph.microsoft.com/.default")) + .block(); + + MessageBox dialog = new MessageBox(getShell(), SWT.ICON_INFORMATION | SWT.OK); + dialog.setText( + BaseMessages.getString(PKG, "SharePointConnectionEditor.TestConnection.Title")); + dialog.setMessage( + BaseMessages.getString(PKG, "SharePointConnectionEditor.TestConnection.Success")); + dialog.open(); + } catch (Exception error) { + new ErrorDialog( + getShell(), + BaseMessages.getString(PKG, "SharePointConnectionEditor.TestConnection.Title"), + BaseMessages.getString(PKG, "SharePointConnectionEditor.TestConnection.Error"), + error); + } + } +} diff --git a/plugins/tech/microsoft365/src/main/resources/org/apache/hop/vfs/sharepoint/metadatatype/messages/messages_en_US.properties b/plugins/tech/microsoft365/src/main/resources/org/apache/hop/vfs/sharepoint/metadatatype/messages/messages_en_US.properties new file mode 100644 index 00000000000..05d6cc23232 --- /dev/null +++ b/plugins/tech/microsoft365/src/main/resources/org/apache/hop/vfs/sharepoint/metadatatype/messages/messages_en_US.properties @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +SharePointAuthenticationType.ManagedIdentity=Managed identity +SharePointAuthenticationType.Secret=Client secret authentication +SharePointAuthenticationType.Certificate=Client certificate authentication +SharePointConnection.Name=SharePoint Connection +SharePointConnection.Description=Named VFS access to the document libraries of the SharePoint sites of a Microsoft 365, through the Microsoft Graph API +SharePointConnectionEditor.AuthenticationType.Label=Authentication type +SharePointConnectionEditor.Name.Label=Name +SharePointConnectionEditor.Description.Label=Description +SharePointConnectionEditor.ClientId.Label=Client ID +SharePointConnectionEditor.ClientId.Tooltip=Client ID of your application +SharePointConnectionEditor.ClientSecret.Label=Client Secret +SharePointConnectionEditor.ClientSecret.Tooltip=Use this for Client Secret authentication with Microsoft Entra ID (recommended for most scenarios). +SharePointConnectionEditor.PEMCertificate.Label=PEM Certificate +SharePointConnectionEditor.PEMCertificate.Tooltip=Use this for Client Certificate authentication with Microsoft Entra ID (alternative for enhanced security). +SharePointConnectionEditor.TenantId.Label=Tenant ID +SharePointConnectionEditor.TenantId.Tooltip=Microsoft Entra tenant ID +SharePointConnectionEditor.TestConnection.Title=Test connection +SharePointConnectionEditor.TestConnection.Success=Connection to Microsoft SharePoint successful! +SharePointConnectionEditor.TestConnection.Error=Error connecting to Microsoft SharePoint! +SharePointConnectionEditor.Hostname.Label=SharePoint host +SharePointConnectionEditor.Hostname.Tooltip=The SharePoint host of the tenant, for example contoso.sharepoint.com. Optional, only needed when a site is not found because the application can not read the root site of the tenant. diff --git a/plugins/tech/microsoft365/src/main/resources/sharepoint.svg b/plugins/tech/microsoft365/src/main/resources/sharepoint.svg new file mode 100644 index 00000000000..0b01a839799 --- /dev/null +++ b/plugins/tech/microsoft365/src/main/resources/sharepoint.svg @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + diff --git a/plugins/tech/microsoft365/src/main/resources/version.xml b/plugins/tech/microsoft365/src/main/resources/version.xml new file mode 100644 index 00000000000..6be576acae9 --- /dev/null +++ b/plugins/tech/microsoft365/src/main/resources/version.xml @@ -0,0 +1,20 @@ + + + +${project.version} \ No newline at end of file diff --git a/plugins/tech/microsoft365/src/test/java/org/apache/hop/vfs/sharepoint/SharePointFileNameTest.java b/plugins/tech/microsoft365/src/test/java/org/apache/hop/vfs/sharepoint/SharePointFileNameTest.java new file mode 100644 index 00000000000..57f1894dd4e --- /dev/null +++ b/plugins/tech/microsoft365/src/test/java/org/apache/hop/vfs/sharepoint/SharePointFileNameTest.java @@ -0,0 +1,112 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hop.vfs.sharepoint; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.apache.commons.vfs2.FileSystemException; +import org.apache.commons.vfs2.FileType; +import org.junit.jupiter.api.Test; + +class SharePointFileNameTest { + + private static SharePointFileName parse(String uri) throws FileSystemException { + return (SharePointFileName) SharePointFileNameParser.getInstance().parseUri(null, null, uri); + } + + @Test + void pluginHasNoFixedScheme() { + SharePointVfsPlugin plugin = new SharePointVfsPlugin(); + assertEquals(0, plugin.getUrlSchemes().length); + assertNull(plugin.getProvider()); + } + + @Test + void parsesNamedConnectionUri() throws Exception { + SharePointFileName name = parse("m365:///Finance/Documents/reports/2026/daily.csv"); + assertEquals("m365", name.getScheme()); + assertEquals("/Finance/Documents/reports/2026/daily.csv", name.getPath()); + assertEquals(FileType.FILE, name.getType()); + assertEquals("m365:///", name.getRootURI()); + assertEquals("Finance", name.getSiteName()); + assertEquals("Documents", name.getDriveName()); + assertEquals("reports/2026/daily.csv", name.getEncodedItemPath()); + assertTrue(name.isItem()); + } + + @Test + void parsesRoot() throws Exception { + for (String uri : new String[] {"m365://", "m365:///", "m365:/"}) { + SharePointFileName name = parse(uri); + assertEquals("/", name.getPath()); + assertEquals(FileType.FOLDER, name.getType()); + assertEquals(0, name.getLevel()); + assertNull(name.getSiteName()); + assertNull(name.getDriveName()); + assertFalse(name.isItem()); + } + } + + @Test + void sitesAndLibrariesAreNotItems() throws Exception { + SharePointFileName site = parse("m365:///Finance"); + assertEquals("Finance", site.getSiteName()); + assertNull(site.getDriveName()); + assertFalse(site.isItem()); + + SharePointFileName library = parse("m365:///Finance/Shared Documents/"); + assertEquals("/Finance/Shared Documents", library.getPath()); + assertEquals("Shared Documents", library.getDriveName()); + assertEquals("", library.getEncodedItemPath()); + assertFalse(library.isItem()); + } + + @Test + void encodesItemPathForGraph() throws Exception { + SharePointFileName name = parse("m365:///Finance/Documents/Q1 report/50%25 d'été #2+.csv"); + // VFS keeps a literal percent sign escaped in the path + assertEquals("/Finance/Documents/Q1 report/50%25 d'été #2+.csv", name.getPath()); + assertEquals("50% d'été #2+.csv", name.getDecodedBaseName()); + assertEquals(4, name.getLevel()); + assertEquals("Q1%20report/50%25%20d%27%C3%A9t%C3%A9%20%232%2B.csv", name.getEncodedItemPath()); + } + + @Test + void resolvesAgainstBase() throws Exception { + SharePointFileName base = parse("m365:///Finance/Documents"); + SharePointFileNameParser parser = + (SharePointFileNameParser) SharePointFileNameParser.getInstance(); + + SharePointFileName relative = (SharePointFileName) parser.parseUri(null, base, "a/b.txt"); + assertEquals("m365", relative.getScheme()); + assertEquals("/Finance/Documents/a/b.txt", relative.getPath()); + + SharePointFileName absolute = + (SharePointFileName) parser.parseUri(null, base, "/Other/Lib/c.txt"); + assertEquals("/Other/Lib/c.txt", absolute.getPath()); + } + + @Test + void relativeUriNeedsBase() { + assertThrows(FileSystemException.class, () -> parse("no/scheme.txt")); + } +} diff --git a/plugins/tech/pom.xml b/plugins/tech/pom.xml index 8d896a9322c..3ba22339f1e 100644 --- a/plugins/tech/pom.xml +++ b/plugins/tech/pom.xml @@ -49,6 +49,7 @@ gremlin hadoop lakehouse + microsoft365 minio mongodb neo4j