Neues PowerShell Script, SCDPM Fehler beheben.

Lange habe ich an diesem Script gearbeitet bis es soweit war, die erste Version zu veröffentlichen. Wie der Titel schon ahnen lässt, behebt dieses Script alle aktuellen SCDPM Alerts.

Aber im Gegensatz zu anderen Scripten die es von dieser Art schon gibt, mach ich hier nicht einfach die “reommende Action” sondern starte für jeden Alert Type, dementsprechend die Aktion die notwendig ist.

Und ich kann hier schon mal verraten, ich vergrößere auch die SCDPM Volumes, sofern es notwendig ist, aber dazu im Artikel mehr.

Die Features

Hier mal eine Liste, welche Fehler dieses Script in der aktuellen Version 1.0 behebt.

• Replica Volume Size Alert
• Shadow Copy Volume Size Alert
• Error created by primary SCDPM Server
• Tape Errors
• Server or Agent not responding
• Replica inactive
• Job was canceled
• Protectiongroup was modified
• VSS Error

  Also schon eine ganze Meng wie ich finde habe dieses Script bei einigen Installation eingerichtet, und soweit ganz gute Erfahrungen gemacht.

Das Script

Zuerst wird das SCDPM PowerShell Module importiert, danach die Variablen definiert.

#*=============================================
#* IMPORT MODULES
#*=============================================
# Module: DataProtection Manager
Import-Module DataProtectionManager


#*=============================================
#* END IMPORT MODULES
#*=============================================
#*=============================================
#* VARIABLE DECLARATION
#*=============================================
# Variables: Config

# Set Location Path
$WorkingPath = "D:\_Scripts\Resolve\"
$LogfileName="AlertLogging.txt"
$LogPublic="Yes"

$DPMServerName="$env:computername"

$AlertCSV='AllAlerts.csv'
$ExtendedAlertCSV='ExtAllAlerts.csv'
$remainingAlertCSV='remAllAlerts.csv'
$remainingExtendedAlertCSV='remExtAllAlerts.csv'

$PSPre="3"

if ($Host.Version.Major -ge $PSPre) {
    } else {
    Write-Host "Wrong PS Version, you are running Version: " $Host.Version.Major
    Write-Host "You need PowerShell V3"
    Write-Host "Download PowerShell V3: http://www.microsoft.com/en-us/download/details.aspx?id=34595"
    Write-Host "Press Key to exit...."
    $x=$HOST.UI.RawUI.ReadKey("NoEcho,IncludeKeyDown") 
    exit
    }

Danach erfolgt die Definition der Funktionen fürs Logging und noch viel wichtiger für unsere SCDPM Jobs

#*=============================================
#* END VARIABLE DECLARATION
#*=============================================
#*=============================================
#* FUNCTION LISTINGS
#*=============================================
# Functions : Admin
Function Write-Log ($Text, $Value) {
    # Set Variables for Logging 
    $Date=Get-Date -format dd.MM.yyyy-HH:mm:ss
    $Log="$Date - $Text $Value"

    # Check if Path or File exists
    if (!(Test-Path -Path $WorkingPath)) {
        New-Item -Path $WorkingPath -ItemType Directory
    }
    if (!(Test-Path -Path $WorkingPath$LogfileName)) {
        New-Item -Path $WorkingPath$LogfileName -ItemType File
    }

    # Write Log
    add-Content -Path $WorkingPath$LogfileName -Value $Log

    # Write Log to PS if $LogPublic is Yes
    if ($LogPublic -eq "Yes") {
        if ($Log -match "INFO:") {
            Write-Host $Log -ForegroundColor Green
        } elseif ($Log -match "WARNING:") {
            Write-Host $Log -ForegroundColor Yellow
        } elseif ($Log -match "ERROR:") {
            Write-Host $Log -ForegroundColor Red
        } else {
            Write-Host $Log
        }
    }


    # If a Error is present, write to Log
    if ($error.Count -gt "0") {
        $ErrorLog=$Date+' - '+"Error:"+$error
        add-Content -Path $WorkingPath$LogfileName -Value $Log
        $error.clear()
    }
}

# Functions : ResolveAlerts
Function Inactive-Alert ($Alert){
    # Set Alert to inactive if there was no error
    if ($Error[0] -eq $null)
    {
        $Alert.ResolveAlert()
        Write-Log "INFO: Set Alert to inactive: " $Alert.Datasource 
    }
    $Error.Clear()
}

Function Query-DSJobs ($DS) {
    # Query DS if there is an active Job
    Write-Log "Info: Query DS for running job: " $DS.Name 
        
    if ((Get-DPMJob -Datasource $DS -Status InProgress | where {$_.JobCategory -ne "ArchiveFromSC"}).count -ne 0) {
        Write-Log "WARNING: The is a running Job for DS: " $DS.Name
        return $False
    } else {
        Write-Log "INFO: No running Jobs for DS: " $DS.Name
        return $True
    }
}

Function Create-Recoverypoint ($Alert) {
    # Run Create a RecoverPoint for affected DS of Alert
    Write-Log "INFO: Try to run Create-Recoverypoint for Alert: " $Alert.Datasource 

    # Get affected DS of Alert
    $DS=Get-Datasource -DPMServerName  $DPMServerName |  where {$_.DatasourceID -eq $Alert.ObjectID -and $_.CurrentProtectionState -ne "Unprotected"} 
    Write-Log "INFO: Datasoruce is: " $DS.Name

    if ((Query-DSJobs $DS)) {
        Write-Log "INFO: New RecoverPoint for DS: " $DS.Name
        New-RecoveryPoint -Datasource $DS -Disk -DiskRecoveryPointOption withsynchronize | Out-Null
        #Inactive-Alert $Alert
    }       
}

Function Resolve-AgentUnreachableAlert ($Alert) {
    # Try to solve Agent unreachable 
    Write-Log "INFO: Try to run Resolve-AgentUnreachable for Alert: " $Alert.Datasource 

    # Get Server of Alert
    $PS=Get-DPMProductionServer -DPMServerName $DPMServerName | where {$Alert.server -eq $_.Name} -verbose

    Write-Log "INFO: Refresh Server: " $PS.ServerName
    Refresh-DPMProductionServer $PS -Verbose

    #Inactive-Alert $Alert
}

Function Resolve-InvalidState ($Alert) {
    # Run Consistency Check for affected DS of Alert
    Write-Log "INFO: Try to run Resolve-InvalidState for Alert: " $Alert.Datasource 
    
    # Get affected DS of Alert
    $DS=Get-Datasource -DPMServerName  $DPMServerName |  where {$_.DatasourceID -eq $Alert.ObjectID -and $_.CurrentProtectionState -ne "Unprotected"} 
    Write-Log "INFO: Datasoruce is: " $DS.Name
    
    if ((Query-DSJobs $DS)) {
        Write-Log "INFO: ConsistencyCheck for DS: " $DS.Name
        Start-DPMDatasourceConsistencyCheck -Datasource $DS | Out-Null
        #Inactive-Alert $Alert
    }   
}

Function Start-RecommendedAction ($Alert){
    # Run Recomended Actions for Alert
    Write-Log "INFO: Try to run Start-RecommendedAction for Alert: " $Alert.Datasource 

    $Alert.TakeRecommendedAction()
    #Inactive-Alert $Alert
}

Function Resume-Tape ($Alert){
    # Resum Tape Jobs
    Write-Log "INFO: Try to run Resume-Tape for Alert: " $Alert.Datasource 

    $Alert.TakeRecommendedAction($alert.GetAdhocJobsContext()) | Out-Null
    #Inactive-Alert $Alert
}

Function Resolve-ProtectionFailedAlert ($Alert){
    # Resolve Alerts where Protection failed
    Write-Log "INFO: Try to run Resolve-ProtectionFailedAlert for Alert: " $Alert.Datasource 
    
    # Get affected DS of Alert
    $DS=Get-Datasource -DPMServerName  $DPMServerName |  where {$_.DatasourceID -eq $Alert.ObjectID -and $_.CurrentProtectionState -ne "Unprotected"} 
    Write-Log "INFO: Datasoruce is: " $DS.Name

    if ((Query-DSJobs $DS)) {
        Write-Log "INFO: Recomended Action for DS: " $DS.Name
        $Alert.TakeRecommendedAction()

        Write-Log "INFO: Wait 5 Seconds to take recommended Action to effect"
        sleep -Seconds 5

        Resolve-InvalidState $Alert
    }     
}

Function Get-PrimaryDPM ($Alert) {
    # Resolve Alerts on Primary DPM which affecting this DPM
    Write-Log "INFO: Try to run Get-PrimaryDPM for Alert: " $Alert.Datasource 

    # Get affected DS of Alert
    $DS=Get-Datasource -DPMServerName  $DPMServerName |  where {$_.DatasourceID -eq $Alert.ObjectID -and $_.CurrentProtectionState -ne "Unprotected"} 
    Write-Log "INFO: Datasoruce is: " $DS.Name

    # Get Source DPM Server
    $PrimaryDPMServer=$DS.SourceDPMServer
    Write-Log "INFO: The primary DPM Server is: " $PrimaryDPMServer

    # Get Computer affected on Primary DPM Server
    $PrimaryDPMDSComputer=$DS.Computer
    Write-Log "INFO: The affected Server on Primary DPM is: " $PrimaryDPMDSComputer

    # Get DS on primary DPM Server
    $PrimaryDPMDSName=$DS.Name
    Write-Log "INFO: The affected DS on Primary DPM is: " $PrimaryDPMDSName

    # Disconnect from DPM Server
    Disconnect-DPMServer
    Write-Log "INFO: Disconnect from this DPM Server"

    $PrimaryDPMDS=Get-Datasource -DPMServerName $PrimaryDPMServer | where {$_.Computer -eq $PrimaryDPMDSComputer -and $_.Name -eq  $PrimaryDPMDSName}
    Write-Log "INFO: Start ConsistencyCheck on remote DPM Server"
    Start-DPMDatasourceConsistencyCheck -Datasource $PrimaryDPMDS
    Disconnect-DPMServer
}

Function Expand-ReplicaVolume ($Alert){
    # Expand Replica Volume
    Write-Log "INFO: Try to run Expand-ReplicaVolume for Alert: " $Alert.Datasource 

    # Get affected DS of Alert
    $DS=Get-Datasource -DPMServerName  $DPMServerName |  where {$_.DatasourceID -eq $Alert.ObjectID -and $_.CurrentProtectionState -ne "Unprotected"} 
    Write-Log "INFO: Datasoruce is: " $DS.Name

    # Get Protectiongroup of Alert
    $PG=Get-ProtectionGroup -DPMServerName $DPMServerName | where {$_.Name -eq $Alert.pg.Name}

    # Modify ProtectionGroup
    $MPG = Get-ModifiableProtectionGroup $PG
	
	if (!$ds) {
    	Write-Log "ERROR: Failed to get DataSoruce: " $DS.Name
	}
	else{
	    $ReplicaSize = $ds.ReplicaSize

        # Increase new Replica Size with 20%
		$NewReplicaSize = $ReplicaSize * 1.2
        $OldSize = "{0:N2}" -f ($ReplicaSize/1GB)
		$NewSize = "{0:N2}" -f ($NewReplicaSize/1GB)

		Write-Log "Info: Modifying Replica Disk Allocation from $OldSize to $NewSize..."
		Set-DatasourceDiskAllocation -Manual -Datasource $ds -ProtectionGroup $MPG -ReplicaArea $NewReplicaSize
    	Set-ProtectionGroup $MPG
    }
}

Function Expand-RecoveryVolume ($Alert){
    # Expand Recovery Volume
    Write-Log "INFO: Try to run Expand-RecoveryVolume for Alert: " $Alert.Datasource 

    # Get affected DS of Alert
    $DS=Get-Datasource -DPMServerName  $DPMServerName |  where {$_.DatasourceID -eq $Alert.ObjectID -and $_.CurrentProtectionState -ne "Unprotected"} 
    Write-Log "INFO: Datasoruce is: " $DS.Name
    
    # Get Protectiongroup of Alert
    $PG=Get-ProtectionGroup -DPMServerName $DPMServerName | where {$_.Name -eq $Alert.pg.Name}

    # Modify ProtectionGroup
    $MPG = Get-ModifiableProtectionGroup $PG
	
    if (!$ds){
        Write-Log "ERROR: Failed to get DataSoruce: " $DS.Name
	}else {
		$SCSize = $ds.ShadowCopyAreaSize

        # Increase new SchadowCopy Size with 20%
		$NewSCSize = $SCSize * 1.2
		$OldSize = "{0:N2}" -f ($SCSize/1GB)
		$NewSize = "{0:N2}" -f ($NewSCSize/1GB)

		Write-Log "INFO: Modifying Recovery Point Disk Allocation from $OldSize to $NewSize..."
		Set-DatasourceDiskAllocation -Manual -Datasource $ds -ProtectionGroup $MPG -ShadowCopyArea $NewSCSize
		Set-ProtectionGroup $MPG
    }			
}

Danach folgt das Herzstück, hier passiert die Magie.

#*=============================================
#* END FUNCTION LISTINGS
#*=============================================
#*=============================================
#* SCRIPT BODY
#*=============================================
# Get all Alerts for Errorhandling
$Alerts=Get-DPMAlert | Sort-Object Priority 

# Get all allerts for documentation
Get-DPMAlert | Sort-Object Priority | select * | Export-Csv -Path  $WorkingPath$AlertCSV -Append
#Get-DPMAlert | Sort-Object Priority | select -ExpandProperty ErrorInfo | Export-Csv -Path  $WorkingPath$ExtendedAlertCSV -Append


foreach ($Alert in $Alerts) 
{
    
    #* COMMUNICATION
    #*============================================= 
    if ($Alert.Type -eq "AgentUnreachableAlert"){
        # Host or Agent not reachable   
        Resolve-AgentUnreachableAlert $Alert
    }

    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "RMAgent") {
        Resolve-AgentUnreachableAlert $Alert
        Resolve-InvalidState $Alert
    }
    
    if ($Alert.Type -eq "AgentUnreachableAlert" -and $Alert.DetailedErrorInfo.code -match "AMHostUnreachable"){
        Resolve-AgentUnreachableAlert $Alert

    }
        
    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "RmDCOMAgentCommunicationError") {
        # Commuinication failed
        Resolve-AgentUnreachableAlert $Alert
        Resolve-InvalidState $Alert
    }

    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "RmAgentNotResponding"){
        # Commuinication failed
        Resolve-AgentUnreachableAlert $Alert
        Resolve-InvalidState $Alert
    }
        
    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "RmAgentCommunicationError"){
        # Commuinication failed
        Resolve-AgentUnreachableAlert $Alert
        Resolve-InvalidState $Alert
    }
        
    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "RmDCOMAgentCommunicationError"){
        # Commuinication failed
        Resolve-AgentUnreachableAlert $Alert
        Resolve-InvalidState $Alert
    }
    
    if ($Alert.Type -eq "SynchronizationFailedAlert" -and $Alert.DetailedErrorInfo.code -match "RmDCOMAgentCommunicationError"){
        # Commuinication failed
        Resolve-AgentUnreachableAlert $Alert
        Resolve-InvalidState $Alert
    }


    #* VOLUMES
    #*=============================================    
    if ($Alert.Type -eq "ShadowCopyConsolidationRequired" -and $Alert.DetailedErrorInfo.code -match "ShadowCopyInsufficientStorage"){
        # not Enough Space for RecoveryVolume
        Expand-RecoveryVolume $Alert
        Resolve-InvalidState $Alert
    } 
     
    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "RmNotEnoughSpaceOnReplica") {
        # not Enough Space for ReplicaVolume
        Expand-ReplicaVolume $Alert
        Resolve-InvalidState $Alert
    } 
    
    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "PrmShadowcopyAreaFull") {
        # not Enough Space for RecoveryVolume
        Expand-RecoveryVolume $Alert
        Resolve-InvalidState $Alert
    } 


    #* REPLICA INVALID
    #*=============================================    
    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "ShadowCopyInProgress"){
       # Replica inactive
       Resolve-InvalidState $Alert
    }

    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "ShadowCopyInProgress"){
       # Replica inactive
       Resolve-InvalidState $Alert
    }
        
    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "PrmConfigureProtectionNeeded"){
       # Replica inaktive
       Resolve-InvalidState $Alert
    }
        
    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "CancelledOnRestart"){
       # Replica inaktive in case of a reboot
       Resolve-InvalidState $Alert
    }
        
    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "ReplicasInvalidAfterUpgrade"){
        # Replic inaktive after upgrade
        Resolve-AgentUnreachableAlert $Alert
        Resolve-InvalidState $Alert
    }
           
    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "EngineCancelledJob"){
        # Job was canceld
        Resolve-AgentUnreachableAlert $Alert
        Resolve-InvalidState $Alert
    }

    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "RmCannotApplyChanges"){
        # Changes cannot be applied
        Create-Recoverypoint $Alert
    } 

    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "PrmUnableToCleanupReplica"){
        # Failed to clean up old incremental Backups
        Resolve-InvalidState $Alert
    }

    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "SimilarTaskExistsForDatasource"){
        # Similar Job
        #Inactive-Alert $Alert
    } 


    #* SHADOWCOPY INVALID
    #*=============================================
    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "PrmVssErrorRetryable"){
        # VSS Error
        Resolve-InvalidState $Alert
    }


    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "PrmExchangeLogChainBroken"){
        # VSS Error
        Resolve-InvalidState $Alert
    }


    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "PrmExchangeAnotherBackupInProgress"){
        # VSS Error
        Resolve-InvalidState $Alert
    }
     
     
    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "ShadowCopyCreationSkipped"){
        # No Changes or no Sync
        Create-Recoverypoint $Alert
    } 
    
    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "EngineCancelledJob"){
        # Protectiongroup was modified
        Create-Recoverypoint $Alert
    } 

    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "CancelledOnRestart"){
        # Replica inactive in case of a reboot
        Resolve-InvalidState $Alert
    }


    #* TAPES
    #*=============================================
    if ($Alert.Type -eq "BackupToTapeFailedAlert") {
        # Tape Backup Failed
        Resume-Tape $Alert
        sleep -Seconds 5
    } 
    
    if ($Alert.Type -eq "DetailedInventoryFailedAlert"){
        # Tape Detailed Inventory failed
        Start-RecommendedAction $Alert
    } 


    #* PRIMARY DPM
    #*=============================================
    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "PrmNonRetryableWriterErrorForDR"){
        # Primary DPM has invalid replica, try to do a replica on primary
        Get-PrimaryDPM $Alert
    }


    #* OTHER STUFF
    #*=============================================
    if ($Alert.Type -eq "ConfigureProtectionFailedAlert") {
       Resolve-ProtectionFailedAlert $Alert
    }

    if ($Alert.Type -eq "RunVerificationJob" -and $Alert.DetailedErrorInfo.code -match "RmCannotApplyChanges") {
        # Replica inaktive in case of Sync error, File Exists
        Resolve-InvalidState $Alert
    }

    if ($Alert.Type -eq "ShadowCopyConsolidationRequired" -and $Alert.DetailedErrorInfo.code -match "ShadowCopyInProgress") {
        # alert can be set to inactive
        #Inactive-Alert $Alert
    }

    if ($Alert.Type -eq "ShadowCopyFailedAlert" -and $Alert.DetailedErrorInfo.code -match "PrmSQLLogChainBroken") {
        # SQL Log Chain error
        Resolve-InvalidState $Alert
    }

    if ($Alert.Type -eq "ReplicaInInvalidStateAlert" -and $Alert.DetailedErrorInfo.code -match "FilterTrackingFailed") {
        # Filtertracking Failed in case of reboot
        Resolve-InvalidState $Alert
    }
    

    if ($Alert.Type -eq "ShadowCopyFailure" -and $Alert.DetailedErrorInfo.code -match "RmAgentInUnexpectedState") {
        # Unexpected State of Agent
        Create-Recoverypoint $Alert
    }



    if ($Alert.Type -eq "ShadowCopyFailure" -and $Alert.DetailedErrorInfo.code -match "RmAgentInUnexpectedState") {
        # Unexpected State of Agent
        Create-Recoverypoint $Alert
    }
    
    if ($Alert.Type -eq "ShadowCopyFailure" -and $Alert.DetailedErrorInfo.code -match "RmAgentInUnexpectedState") {
        # Unexpected State of Agent
        Create-Recoverypoint $Alert
    }


}


# Get all allerts for documentation which are not solved
Get-DPMAlert | Sort-Object Priority | select * | Export-Csv -Path  $WorkingPath$remainingAlertCSV -Append
#Get-DPMAlert | Sort-Object Priority | select -ExpandProperty ErrorInfo | Export-Csv -Path  $WorkingPath$remainingExtendedAlertCSV -Append


#*=============================================
#* END SCRIPT BODY
#*=============================================

TechNet Gallery Download

Download auf der TechNet Gallery: https://gallery.technet.microsoft.com/Resolve-SCDPM-Alerts-705b4e5f

Solltet ihr Fragen, Wünsche oder Probleme mit dem Script haben, schreibt mir ein Kommentar

Alle meine TechNet Gallery Downloads findet ihr hier: 1jrYQoA

BITTE BEWERTET MEINE DOWNLOADS IN DER TECHNET GALLERY UND SAGT MIR WAS IHR EUCH VON DER NÄCHSTEN VERSION WÜNSCHT.

Michael Seidl aka Techguy