There are many examples available that shows how to bulk migrate or upload files to SharePoint but none of those are providing a method to migrate with metadata. The script below can be used to migrate complicated folder hierarchy to be uploaded to SharePoint Online or On-Premises. The script is based on the the effort on Technet example https://gallery.technet.microsoft.com/PowerShell-Bulk-Upload-b9e9d600. The same script can be modified to upload large amount of metadata. You just need to export the files list to a CSV and then read values after the upload. I will upload the other example later.
[CmdletBinding()]
param(
[Parameter(Mandatory=$false,Position=1)]
[String]$Credentials,
[Parameter(Mandatory=$True, Position=3)]
[String]$SiteURL,
[Parameter(Mandatory=$True, Position=4)]
[String]$DocLibName,
[Parameter(Mandatory=$True, Position=5)]
[String]$Folder,
[Parameter(Mandatory=$False, Position=6)]
[Switch]$Checkin,
[Parameter(Mandatory=$False, Position=7)]
[Switch]$O365
)
<#
Add references to SharePoint client assemblies and authenticate to Office 365 site – required for CSOM
Add-Type -Path “C:\Program Files\Common Files\Microsoft Shared\Web Server Extensions\15\ISAPI\Microsoft.SharePoint.Client.dll”
Add-Type -Path “C:\Program Files\Common Files\Microsoft Shared\Web Server Extensions\15\ISAPI\Microsoft.SharePoint.Client.Runtime.dll”
#>
Add-Type -Path “C:\Program Files\Common Files\Microsoft Shared\Web Server Extensions\15\ISAPI\Microsoft.SharePoint.Client.dll” -ErrorAction SilentlyContinue
Add-Type -Path “C:\Program Files\Common Files\Microsoft Shared\Web Server Extensions\15\ISAPI\Microsoft.SharePoint.Client.Runtime.dll” -ErrorAction SilentlyContinue
Add-Type -Path ([System.Reflection.Assembly]::LoadWithPartialName(“Microsoft.SharePoint.Client”).location)
Add-Type -Path ([System.Reflection.Assembly]::LoadWithPartialName(“Microsoft.SharePoint.Client.runtime”).location)
function GetUserLookupString{
[CmdletBinding()]
param($context, $userString)
try{
$user = $context.Web.EnsureUser($userString)
$context.Load($user)
$context.ExecuteQuery()
# The “proper” way would seem to be to set the user field to the user value object
# but that does not work, so we use the formatted user lookup string instead
#$userValue = New-Object Microsoft.SharePoint.Client.FieldUserValue
#$userValue.LookupId = $user.Id
$userLookupString = “{0};#{1}” -f $user.Id, $user.LoginName
}
catch{
Write-Host “Unable to ensure user ‘$($userString)’.” -ErrorAction SilentlyContinue
$userLookupString = $null
}
return $userLookupString
}
<#
Define Functions
#>
<#
Upload File – This function performs the actual file upload
#>
function UploadFile($DestinationFolder, $File)
{
#Get the datastream of the file, assign it to a variable
$FileStream = New-Object IO.FileStream($File.FullName,[System.IO.FileMode]::Open)
#Create an instance of a FileCreationInformation object
$FileCreationInfo = New-Object Microsoft.SharePoint.Client.FileCreationInformation
#Indicate whether or not you would like to overwrite files in the event of a conflict
$FileCreationInfo.Overwrite = $True
#Make the datastream of the file you wish to create equal to the datastream of the source file
$FileCreationInfo.ContentStream = $FileStream
#Make the URL of the file equal to the $File variable which was passed to the function. This will be equal to the source file name
$FileCreationInfo.url = $File
#Add the file to the destination folder which was passed to the function, using the FileCreationInformation supplied. Assign this to a variable so that it can be loaded into context.
$Upload = $DestinationFolder.Files.Add($FileCreationInfo)
if($Checkin)
{
$Context.Load($Upload)
$Context.ExecuteQuery()
if($Upload.CheckOutType -ne “none”)
{
$Upload.CheckIn(“Checked in by Administrator”, [Microsoft.SharePoint.Client.CheckinType]::MajorCheckIn)
}
}
$Context.Load($Upload)
$Context.ExecuteQuery()
Write-Host “Adding Metadata to File Item…$($File.FullName)”
$listItem = $Upload.ListItemAllFields
$Context.Load($listItem)
#$Context.ExecuteQuery()
$Context.ExecuteQuery()
$Item = $List.GetItemById($listItem.Id);
$Context.Load($Item)
$Owner = $File.GetAccessControl().Owner
$LastModifiedBy =Get-ChildItem $File.FullName | Sort {$_.LastWriteTime} | select -last 1 | foreach {$a=$_;$b=Get-Acl $_.FullName; Add-Member -InputObject $b -Name “LastWriteTime” -MemberType NoteProperty -Value $a.LastWriteTime;$b}
if($Owner -eq “BUILTIN\Administrators”)
{
$Author = “hp\spadmin”
}
if($LastModifiedBy.Owner -eq “BUILTIN\Administrators”)
{
$Editor = “hp\spadmin”
}
$Author = GetUserLookupString $Context $Author
if($Author)
{
$Item[“Author”] = $Author
}
$Editor = GetUserLookupString $Context $Editor
if($Editor)
{
$Item[“Editor”] = $Editor
}
[System.DateTime]$CreatedOnDate = $File.CreationTime
[System.DateTime]$ModifiedOnDate = $File.LastWriteTime
$Item[“Created”] = $CreatedOnDate
$Item[“Modified”] = $ModifiedOnDate
try
{
$Item.Update()
$Context.ExecuteQuery()
}
catch
{ Write-Host “Exception Occured in reading list item properties… But script will continue” }
}
<#
Create Folder Function.
#>
function PopulateFolder($ListRootFolder, $FolderRelativePath, $FolderFullPath)
{
#split the FolderRelativePath passed into chunks (between the backslashes) so that we can check if the folder structure exists
$PathChunks = $FolderRelativePath.substring(1).split(“\”)
#Make sure we start with a fresh WorkingFolder for every folder passed to the function
if($WorkingFolder)
{
Remove-Variable WorkingFolder
}
#Start with the root folder of the list, load this into context
$WorkingFolder = $ListRootFolder
$Context.load($WorkingFolder)
$Context.ExecuteQuery()
#Load the folders of the current working folder into context
$Context.load(($WorkingFolder.folders))
$Context.executeQuery()
#Set the FileSource folder equal to the absolute path of the folder that passed to the function
$FileSource = $Folder + $FolderRelativePath
#Loop through the folder chunks, ensuring that the correct folder hierarchy exists in the destination
foreach($Chunk in $PathChunks)
{
#Check to find out if a subfolder exists in the current folder that matches the patch chunk being evaluated
if($WorkingFolder.folders | ? {$_.name -eq $Chunk})
{
#Log the status to the PowerShell host window
Write-Host “Folder $Chunk Exists in” $WorkingFolder.name -ForegroundColor Green
#Since we will be evaluating other chunks in the path, set the working folder to the current folder and load this into context.
$WorkingFolder = $WorkingFolder.folders | ? {$_.name -eq $Chunk}
$Context.load($WorkingFolder)
$Context.load($WorkingFolder.folders)
$Context.ExecuteQuery()
}
else
{
#If the folder doesn’t exist, Log a message indicating that the folder doesn’t exist, and another message indicating that it is being created
Write-Host “Folder $Chunk Does Not Exist in” $WorkingFolder.name -ForegroundColor Yellow
Write-Host “Creating Folder $Chunk in” $WorkingFolder.name -ForegroundColor Green
#Load the working folder into context and create a subfolder with a name equal to the chunk being evaluated, and load this into context
$Context.load($WorkingFolder)
$Context.load($WorkingFolder.folders)
$Context.ExecuteQuery()
$WorkingFolder= $WorkingFolder.folders.add($Chunk)
$Context.load($WorkingFolder)
$Context.load($WorkingFolder.folders)
$Context.ExecuteQuery()
#$FolderRelativePath = “C:\Demo\Corporate Docs”
$LocalFolder = Get-Item -Path $FolderFullPath
$Web = $Context.Web
$Context.Load($Web)
$Context.ExecuteQuery()
$SPFolder = $Web.GetFolderByServerRelativeUrl($WorkingFolder.ServerRelativeUrl)
$Context.Load($SPFolder)
$Context.Load($SPFolder.ListItemAllFields)
$Context.ExecuteQuery()
$FolderItemId = $SPFolder.ListItemAllFields.Id
$FItem = $List.GetItemById($FolderItemId)
$Context.Load($FItem)
$Context.ExecuteQuery()
#$Query = New-Object Microsoft.SharePOint.Client.CamlQuery
#$Query.ViewXml = “<View Scope=’RecursiveAll’><Query><Where><And><Eq><FieldRef Name=’ContentType’/><Value Type=’Text’>Folder</Value></Eq><Eq><FieldRef Name=’FileLeafRef’/><Value Type=’Text’>” + $Chunk + “</Value></Eq></And></Where></Query></View>”
$FItem[“Created”] = $LocalFolder.CreationTime.ToString()
$FItem[“Modified”] = $LocalFolder.LastWriteTime.ToString()
$CreatedBy = $LocalFolder.GetAccessControl().Owner
if($CreatedBy -eq “BUILTIN\Administrators”)
{
$CreatedBy = “hp\spadmin”
}
$Author = GetUserLookupString $Context $CreatedBy
if($Author)
{
$FItem[“Editor”] = $Author
$FItem[“Author”] = $Author
}
$Context.ExecuteQuery()
try
{
$FItem.Update()
$Context.ExecuteQuery()
}
catch
{ Write-Host “Exception Occured in updating folder properties… But script will continue” }
}
}
#Folder is confirmed existing or created – now it’s time to list all files in the source folder, and assign this to a variable
$FilesInFolder = Get-ChildItem -Path $FileSource | ? {$_.psIsContainer -eq $False}
#For each file in the source folder being evaluated, call the UploadFile function to upload the file to the appropriate location
Foreach ($File in ($FilesInFolder))
{
#Notify the operator that the file is being uploaed to a specific location
Write-Host “Uploading file ” $file.Name “to” $WorkingFolder.name -ForegroundColor Cyan
#Upload the file
UploadFile $WorkingFolder $File
}
}
<#
Bind your context to the site collection
#>
$Context = New-Object Microsoft.SharePoint.Client.ClientContext($SiteURL)
<#
Create a credential object using the username and password supplied
#>
if($O365)
{
$Creds = New-Object Microsoft.SharePoint.Client.SharePointOnlineCredentials($Credentials.UserName,$Credentials.Password)
}
else
{
$Creds = [System.Net.CredentialCache]::DefaultCredentials
#$Creds = New-Object System.Net.NetworkCredential($Credentials.UserName,$Credentials.Password)
}
#if($O365)
#{
# $Creds = New-Object Microsoft.SharePoint.Client.SharePointOnlineCredentials($UserName,(ConvertTo-SecureString $Password -AsPlainText -Force))
#}
#else
#{
# #$Creds = New-Object System.Net.NetworkCredential($UserName, (ConvertTo-SecureString $Password -AsPlainText -Force))
#}
<#
Set the credentials that are used in the context.
#>
$Context.Credentials = $Creds
<#
Retrieve the library, and load it into the context
#>
$List = $Context.Web.Lists.GetByTitle($DocLibName)
$Context.Load($List)
$Context.ExecuteQuery()
$List.Fields.GetByInternalNameOrTitle(“Author”).ReadOnlyField = $true
$List.Fields.GetByInternalNameOrTitle(“Editor”).ReadOnlyField = $true
$List.Fields.GetByInternalNameOrTitle(“Created”).ReadOnlyField = $true
$List.Fields.GetByInternalNameOrTitle(“Modified”).ReadOnlyField = $true
#$List.EnableVersioning = $True
$List.Update()
$Context.ExecuteQuery()
#Get a recursive list of all folders beneath the folder supplied by the operator
$AllFolders = Get-ChildItem -Recurse -Path $Folder |? {$_.psIsContainer -eq $True}
#Get a list of all files that exist directly at the root of the folder supplied by the operator
$FilesInRoot = Get-ChildItem -Path $Folder | ? {$_.psIsContainer -eq $False}
#Upload all files in the root of the folder supplied by the operator
Foreach ($File in ($FilesInRoot))
{
#Notify the operator that the file is being uploaded to a specific location
Write-Host “Uploading file ” $File.Name “to” $DocLibName -ForegroundColor Cyan
#Upload the file
UploadFile($list.RootFolder) $File
}
#Loop through all folders (recursive) that exist within the folder supplied by the operator
foreach($CurrentFolder in $AllFolders)
{
#Set the FolderRelativePath by removing the path of the folder supplied by the operator from the fullname of the folder
$FolderRelativePath = ($CurrentFolder.FullName).Substring($Folder.Length)
#Call the PopulateFolder function for the current folder, which will ensure that the folder exists and upload all files in the folder to the appropriate location
PopulateFolder ($list.RootFolder) $FolderRelativePath $CurrentFolder.FullName
}
#Calling the script for SharePoint on-premises
$Creds = Get-Credential
.\BulkUploadSharePointCSOM.ps1 -SiteURL “https://portal.hp.com” -DocLibName “Metadata” -Folder “C:\Demo”
#Calling the script for SharePoint online
$Creds = Get-Credential
.\BulkUploadSharePointCSOM.ps1 -SiteURL “https://portal.hp.com” -DocLibName “Metadata” -Folder “C:\Demo” -Credentials $Creds -Checkin -O365
Jerry Yasir's Blog 