Submitted byCategory
Review Cycle
.
Public
Joachim Mutter/sysarc
on 05/30/2008 at 04:53 PM
SSiS\Code, SSiS\Doku, SSiS\AAA-Relevant

Logging and Variables

Variables in SSiS are a never ending story. Sometimes you think you have everything understood, but a few days later, you run into
an error handling variables.

Last modifications : 21.07.2011
Run into errors ralated to Variable Deadlocking. After houres of research, I found out how exectly that stuff works in boith the Control-
and the Dataflow Last modifications are related to Lock/Unlock problems. So in case of an error and a variable was locked before, but
not unlocked, you could run into DeadLock situations. I found a solution after some trouble and do the Unlock which have be done in
Read / Write functions in the finally section, so it guarantied, that each variabel which was locked, is afterwards unlocked.

ControlFlow

Parent package configurations can be used to produce an almost opposite behaviour. They are used to receive values from a parent package IF the package has been called by a parent package. I like to use the analogy that this is like passing ByValue whereas the scenario explained above is a bit like passing ByRef.             If Dts.VariableDispenser.Contains(varName) Then    ' Should we check or should we throw an error in case of no exisitnace?
                If isSystem Then ' Use System way to access variables
                    Call Dts.VariableDispenser.LockOneForRead(varName, vars)        ' Locks only this variable and returns it in the vars collection
                    result = vars(varName)                                          ' Read variable
                    vars.Unlock()                                                   ' And unlock it
                Else ' Script Task collection
                    vars = Dts.Variables                                            ' Get the tasks variable collection, which is already locked,
' donot unlock it, if you want to iterate over it !!!
                    result = vars(varName)                                          ' Read variable
                End If For Each prop As PropertyInfo In me.Variables.GetType().GetProperties()
      mMain.VariableDispenser.LockForRead(prop.Name)
Next
      Call mMain.VariableDispenser.GetVariables(mVars)
' ********************************************************************
' ********************************************************************
'
'   This classes provide a standart way to access variables and connection
'   strings and provide you with a standard logging mechanism.
'      
'   Author      Joachim Mutter (joachim.mutter@sysarc.de)
'               All rights reserved!
'   Created     2006/10/10
'   Modified    2011/07/21
'  
' ********************************************************************
' ********************************************************************
' Additional imports
'   Imports Microsoft.SqlServer.Dts.Runtime.Wrapper
'   Imports Microsoft.SqlServer.Dts.Pipeline.Wrapper
'   Imports System.Reflection
'   Imports System.Collections.Generic
'
' Standard Logging in ControlFlow
'   sub Main
'       mLog = New MyLog(Dts, "Logging")
'       MsgBox(String.Format("User Variables: {0}", mLog.UserVariables(vbCr))) ' Example
' ********************************************************************
' ********************************************************************
' Standard Logging in ControlFlow
'    Public Overrides Sub PreExecute()
'        MyBase.PreExecute()
'        mLog = New MyLog(Me, "Logging")
'        MsgBox(String.Format("User Variables: {0}", mLog.UserVariables(vbCr))) ' Example
'    End Sub
' ********************************************************************
' ********************************************************************

#Const FLOWTYPE = 1         ' ControlFlow
'#Const FLOWTYPE = 2         ' DataFlow


Class MyLog
    Inherits MyVariables

    Private mSysLogLevel As Integer
    Private mID As Integer = 0
    Private mShowMsgbox As Boolean = False
    Private mStackTrace As String = ""
    Private mConnectionString As Dictionary(Of String, String)

    Public Const LOG_Error As Integer = &H1
    Public Const LOG_Warning As Integer = &H2
    Public Const LOG_Info As Integer = &H4
    Public Const LOG_Verbose As Integer = &H8
    Public Const LOG_MsgBox As Integer = &H10
    Public Const LOG_SQLStatement As Integer = &H20
    Public Const LOG_UserVars As Integer = &H100
    Public Const LOG_SysVars As Integer = &H200

    Public ReadOnly Property ConnectionString(ByVal szName As String) As String
        Get
            ConnectionString = ""
            mConnectionString.TryGetValue(szName, ConnectionString)
            If ConnectionString.Length <= 0 Then
                For Each ConnectionString In mConnectionString.Values
                    Me.log(LOG_Warning, "Cannot find Connection '{0}', use the first found one : '{1}'", szName, ConnectionString)
                    Exit For
                Next
            End If
        End Get
    End Property

    '_____________________________________________________________________
    ' Constructor
    '_____________________________________________________________________
#If FLOWTYPE = 1 Then         '====== ControlFlow
    Sub New(ByRef main As Microsoft.SqlServer.Dts.Tasks.ScriptTask.ScriptObjectModel, ByVal LogLevelVariable As String)
        MyBase.New(main)
        Try
            mEvents = mMain.Events
            mSysLogLevel = CInt(Me.Read(LogLevelVariable, "0"))     ' Get the logLevel
            Me.GetConnectionString()                                ' Get all ConnectionStrings from all available Connections
        Catch ex As Exception
            Me.FireError(1, Me.ToString, ex.ToString)
        End Try
    End Sub
#Else
    Sub New(ByRef main As ScriptMain, ByVal LogLevelVariable As String)
        MyBase.New(main)
        Try
            mEvents = mMain.ComponentMetaData
            mSysLogLevel = CInt(Me.Read(LogLevelVariable, "0"))     ' Get the logLevel
            Me.GetConnectionString()                                ' Get all ConnectionStrings from all available Connections
        Catch ex As Exception
            Me.FireError(1, Me.ToString, ex.ToString)
        End Try
    End Sub
#End If

    '_____________________________________________________________________
    ' Property SysLogLevel
    '_____________________________________________________________________
    Public Property SysLogLevel() As Integer
        Get
            SysLogLevel = mSysLogLevel
        End Get
        Set(ByVal value As Integer)
            mSysLogLevel = value
        End Set
    End Property

    '_____________________________________________________________________
    ' Some explicit properties
    '_____________________________________________________________________
    ReadOnly Property PackageName() As String
        Get
            Return Me.Read("System::PackageName").ToString
        End Get
    End Property
    ReadOnly Property Taskname() As String
        Get
            Return Me.Read("System::TaskName").ToString
        End Get
    End Property
    ReadOnly Property FullTaskname() As String
        Get
            Return Me.StackTrace + "." + Me.Read("System::TaskName").ToString
        End Get
    End Property
    ReadOnly Property isSysVars() As Boolean
        Get
            isSysVars = ((mSysLogLevel And LOG_SysVars) > 0)
        End Get
    End Property
    ReadOnly Property isUserVars() As Boolean
        Get
            isUserVars = ((mSysLogLevel And LOG_UserVars) > 0)
        End Get
    End Property
    ReadOnly Property isMsgBox() As Boolean
        Get
            isMsgBox = ((mSysLogLevel And LOG_MsgBox) > 0)
        End Get
    End Property
    Public ReadOnly Property StackTrace() As String
        Get
            If mStackTrace.Length = 0 Then
                mStackTrace = Me.Read("ErrorTask").ToString()
                If mStackTrace.Length > 0 Then mStackTrace = Me.Read("System::PackageName").ToString() + "/" + mStackTrace Else mStackTrace = Me.Read("System::PackageName").ToString()
            End If
            Return mStackTrace
        End Get
    End Property
    Public ReadOnly Property UserVariables(Optional ByVal sep As String = ",") As String
        Get
            Return Me.VariableList(sep).ToString()
        End Get
    End Property

    '_____________________________________________________________________
    ' Function setID
    ' Description
    '   This function sets the internal id field which will be used instead
    '   of the LogLevel value for the Identifier printed out
    '   The value will be resetted after each Write function call.
    '   We return a reference of us back to the caller to mkae suche constructs possible :
    '   myInstance.SetID(&h1000).Write(MyClass.LogError, "MySource", "Test")
    '_____________________________________________________________________
    Function setID(ByVal id As Integer) As MyLog
        mID = id
        Return (Me)
    End Function

    '___________________________________________________________________
    ' Build the connection string from OLEDB to .net key value pairs
    '___________________________________________________________________
    Private Sub GetConnectionString()
        Dim isSQLAuthenticate As Boolean
        Dim vPrevent() As String = {"provider", "auto translate", "persist security info"}
        Dim conResult As StringBuilder
        Const CONST_SEPARATOR As String = ";"

        mConnectionString = New Dictionary(Of String, String)                                       ' Handle multiple connection strings

#If FLOWTYPE = 1 Then           '====== ControlFlow
        For Each conMan As ConnectionManager In mMain.Connections
            Dim szConnectionString = conMan.ConnectionString
#Else                           '====== DataFlow
        For Each conMan As IDTSRuntimeConnection100 In mMain.ComponentMetaData.RuntimeConnectionCollection
            Dim szConnectionString = conMan.ConnectionManager.ConnectionString
#End If
            conResult = New StringBuilder
            For Each szPart In Split(szConnectionString , CONST_SEPARATOR)
                Dim keyValues() As String = Split(LCase(szPart), "=")                               ' Split the KeyValue pair
                If UBound(keyValues) > 0 And StrComp("data source", Trim(keyValues(0))) = 0 Then
                    If StrComp(".", Trim(keyValues(1))) <> 0 Then
                        isSQLAuthenticate = True                                                    ' local server => no autentication
                    End If
                End If
                If Array.IndexOf(vPrevent, Trim(keyValues(0))) < 0 Then
                    conResult.Append(szPart & CONST_SEPARATOR)                                      ' Is this a allowed part?
                End If
            Next

            If isSQLAuthenticate And InStr(LCase(conResult.ToString()), "password=") = 0 Then       ' Should we use authentication?
                conResult.Append("Password=" & Me.Read("Password").ToString & CONST_SEPARATOR)      ' Get the the password
            End If
            mConnectionString.Add(conMan.Name, conResult.ToString)
        Next
        For Each foundConnections As KeyValuePair(Of String, String) In mConnectionString
            Call Me.log(MyLog.LOG_Verbose, "", "Found Connections : {0} = '{1}'", foundConnections.Key, foundConnections.Value)
        Next
    End Sub

    '_____________________________________________________________________
    ' Function log
    ' Description
    '   This function logs the given Message, which could be build up like the
    '   WriteFunction in the System.Console with optional paramters {0} .. {n}
    ' LogLevel : Print out only if this level is included in the binary SysLogLevel
    ' Message : Message string
    ' Source : Which Task should be print out in the message
    ' parameter : Optional Values to build into the message
    '_____________________________________________________________________
    Sub log(ByVal logLevel As Integer, ByVal source As String, ByVal message As String, ByVal ParamArray parameter() As Object)
        Dim Id As Integer

        If (logLevel And mSysLogLevel) > 0 Then
            If parameter.Length > 0 Then message = New StringBuilder("").AppendFormat(message, parameter).ToString
            If mID = 0 Then Id = logLevel Else Id = mID ' if we had no internal id, use the external
            If source.Length = 0 Then source = Me.Taskname ' If we had no source, get it from system
            message = message.Replace("\n", vbCrLf)

            If logLevel = LOG_Info Or logLevel = LOG_Verbose Then
                Me.FireInformation(Id, source, message)
            ElseIf logLevel = LOG_Warning Then
                Me.FireWarning(Id, source, message)
            ElseIf logLevel = LOG_Error Then
                Me.FireError(Id, source, message)
            ElseIf logLevel = LOG_MsgBox Then
                MsgBox(String.Format("Stack : {0}\n{1}", Me.FullTaskname, message.Replace(",", vbCrLf)).Replace("\n", vbCrLf), , Me.PackageName + " :: ErrorID = " + CStr(Id))
            ElseIf logLevel = LOG_UserVars Then
                Me.FireInformation(Id, source, Me.UserVariables())
            End If
            mID = 0                                         ' Reset the internal ID
        End If
    End Sub

    Sub logError(ByVal Message As String, ByVal ex As Exception, Optional ByVal param1 As String = "")
        If Not ex.InnerException Is Nothing Then
            If Len(param1) > 0 Then
                Me.log(LOG_Error, "", "{0} {1}\n{2}\n{3}\n{4}", Message, ex.ToString(), ex.InnerException.ToString(), param1)
            Else
                Me.log(LOG_Error, "", "{0} {1}\n{2}", Message, ex.ToString(), ex.InnerException.ToString())
            End If
        Else
            If Len(param1) > 0 Then
                Me.log(LOG_Error, "", "{0} {1}\n{2}", Message, ex.ToString(), param1)
            Else
                Me.log(LOG_Error, "", "{0} {1}", Message, ex.ToString())
            End If
        End If
    End Sub

    Function sprintf(ByVal message As String, ByVal ParamArray parameter() As Object) As String
        Dim stringBuilder As New StringBuilder("")
        sprintf = stringBuilder.AppendFormat(message, parameter).ToString()
    End Function

End Class

'___________________________________________________________
' Class for handling variables in Data-/ControlFlow
' This should be instanciated the very fist step
'___________________________________________________________
Class MyVariables
    Private mVarNames As List(Of String) = New List(Of String)
#If FLOWTYPE = 1 Then         '====== ControlFlow

#If VBC_VER = 8.0 Then
    Protected mEvents As IDTSComponentEvents
#Else
    Protected mEvents As Microsoft.SqlServer.Dts.Tasks.ScriptTask.EventsObjectWrapper        ' SQLServer 2008
#End If

    Protected mMain As Microsoft.SqlServer.Dts.Tasks.ScriptTask.ScriptObjectModel
    Protected mVars As Variables
#Else
    Protected mMain As ScriptMain
#If VBC_VER = 8.0 Then
        Protected mEvents As IDTSComponentMetaData90
        Protected mVars As IDTSVariables90
#Else
    Protected mEvents As IDTSComponentMetaData100
    Protected mVars As IDTSVariables100
#End If
#End If

    Public Sub FireInformation(ByVal code As Integer, ByVal subComponent As String, ByVal description As String)
        mEvents.FireInformation(code, subComponent, description, "", 0, True)
    End Sub
    Public Sub FireWarning(ByVal code As Integer, ByVal subComponent As String, ByVal description As String)
        mEvents.FireWarning(code, subComponent, description, "", 0)
    End Sub
    Public Sub FireError(ByVal code As Integer, ByVal subComponent As String, ByVal description As String)
#If FLOWTYPE = 1 Then         '====== ControlFlow
        mEvents.FireError(code, subComponent, description, "", 0)
#Else
        mEvents.FireError(code, subComponent, description, "", 0, True)
#End If
    End Sub

    '___________________________________________________________
    ' Constructor
    '___________________________________________________________
#If FLOWTYPE = 1 Then         '====== ControlFlow
    Sub New(ByRef main As Microsoft.SqlServer.Dts.Tasks.ScriptTask.ScriptObjectModel)  '====== ControlFlow

        mMain = main             '         '====== ControlFlow
        mEvents = mMain.Events

#Else
    Sub New(ByVal main As ScriptMain) '====== DataFlow

        mMain = main           '         '====== DataFlow
        mEvents = mMain.ComponentMetaData

#End If

        Call Me.InitVariableNames()     ' First of all, get the ReadOnly and ReadWriteVariables of the task and all other defined variables (ErrorAdditionalVariables)

        Me.VariableNames = Me.Read("ErrorAdditionalVariables", "").ToString()
        Me.VariableNames = "ErrorLevel,ErrorCatchLevel,ErrorDisableDBLogging,ErrorChild,ErrorRethrow,ErrorDoNotHandle,ErrorTask"
    End Sub

    '___________________________________________________________
    ' Returns back the initial variable names
    ' or if compareName is given, the we return back each name
    ' which contains CompareName
    '___________________________________________________________
    Property VariableNames(Optional ByVal CompareName As String = "") As String
        Get
            If CompareName.Length = 0 Then
                VariableNames = String.Join(",", mVarNames.ToArray())
            Else
                Dim result As String = ""
                For Each varName As String In mVarNames
                    If varName.Contains(CompareName) Then
                        If result.Length = 0 Then result = varName Else result = result & "," & varName
                    End If
                Next
                VariableNames = result
            End If
        End Get
        Set(ByVal value As String)
            Dim v() As String = value.Split(","c)
            For Each varName As String In value.Split(","c)
                If Not mVarNames.Contains(varName) Then
                    mVarNames.Add(varName)
                End If
            Next
            Me.getVars()
        End Set
    End Property

    '___________________________________________________________
    ' Returns back the initial variable collection
    '___________________________________________________________
    ReadOnly Property Variables() As Variables
        Get
            Variables = Me.getVars()
        End Get
    End Property

    '___________________________________________________________
    ' Returns all given varibales, also the additional ones back
    ' as a stringBuilder variable.
    '___________________________________________________________
    Public ReadOnly Property VariableList(Optional ByVal sep As String = ",") As StringBuilder
        Get
            Dim sb As StringBuilder = New StringBuilder()
            Dim localSep As String = ""
            Try
                Me.getVars()        ' Rebuild the variables collection
                For Each varName As String In mVarNames
                    If varName.Length > 0 Then
                        If mVars.Contains(varName) Then
                            Dim var As Variable = mVars(varName)
                            sb.Append(String.Format("{0} = {1} [DataType: {2}; EvaluateAsExpression: {3}]", var.QualifiedName, var.Value.ToString(), var.DataType.ToString(), var.EvaluateAsExpression.ToString())).Append(localSep)
                        Else
                            sb.AppendFormat("{0} doesn't exist", varName).Append(localSep)
                        End If
                        localSep = sep
                    End If
                Next
            Catch ex As Exception
                sb.Append("Error during iteration of variables : " + ex.Message)
            Finally
                Me.Unlock()
            End Try
            Return sb
        End Get
    End Property

    '_________________________________________________________________________________
    ' Read variables
    ' Parameter
    '   varname :   Variablename, must be in the read or write section of the task
    '               If you want to query system vars, you must use the syntax 'System::<defined varname>'
    '   checkForExistance : Bool var to check or not before using the vars
    '_________________________________________________________________________________
    Function Read(ByVal varName As String, Optional ByVal DefaultValue As Object = Nothing, Optional ByVal NoExisting As String = "", Optional ByVal checkForExistance As Boolean = True) As Object
        Dim result As Object = ""

        Try
            If Not checkForExistance Or mMain.VariableDispenser.Contains(varName) Then
                Me.Unlock()
                Call mMain.VariableDispenser.LockOneForRead(varName, mVars)
                result = mVars(varName).Value                                            ' Read variable
            Else
                If Not mVarNames.Contains(varName) And varName Like "Config::*" Then
                    mEvents.FireWarning(&H1001, "ReadVariable", "Warning variable '" + varName + "' doen't exist in the collection", "", 0)
                    MsgBox("Cannot found " + varName)
                End If
                If Not DefaultValue Is Nothing Then result = DefaultValue Else result = NoExisting
            End If
        Catch ex As Exception
            mEvents.FireInformation(&H1002, "ReadVariable", "Error reading var " + varName + " : " + ex.Message, "", 0, True)
            Throw ex
        Finally
            Me.Unlock()
        End Try

        If Not DefaultValue Is Nothing Then
            If TypeOf (result) Is String Then
                If Len(Trim(result.ToString)) = 0 Then
                    result = DefaultValue
                End If
            End If
        End If

        Return result
    End Function

    '_________________________________________________________________________________
    ' Write variables
    ' Parameter
    '   varname :   Variablename, must be in the read or write section of the task
    '               If you want to query system vars, you must use the syntax 'System::<defined varname>'
    '   value:      Varibale value as Object to write to
    '   checkForExistance : Bool var to check or not before using the vars
    '_________________________________________________________________________________
    Sub Write(ByVal varName As String, ByVal value As Object, Optional ByVal checkForExistance As Boolean = True)
        Try
            If Not checkForExistance Or mMain.VariableDispenser.Contains(varName) Then
                Me.Unlock()
                Call mMain.VariableDispenser.LockOneForWrite(varName, mVars)
                mVars(varName).Value = value                                            ' Write variable
            Else
                If Not mVarNames.Contains(varName) Then
                    mEvents.FireWarning(&H1001, Me.ToString, "Warning variable '" + varName + "' doen't exist in the collection", "", 0)
                End If
            End If
        Catch ex As Exception
            mEvents.FireInformation(&H1002, Me.ToString, "Error write var " + varName + " : " + ex.Message, "", 0, True)
            Throw ex
        Finally
            Me.Unlock()
        End Try
    End Sub

    '___________________________________________________________
    ' Check, if a variable exists
    '___________________________________________________________
    Function CheckForVariable(ByVal varName As String) As String
        If mMain.VariableDispenser.Contains(varName) Then
            Return varName + " exists"
        Else
            Return varName + " doesn't exists"
        End If
    End Function

    '___________________________________________________________
    ' Get the initial Variable names
    ' This must be the first call, becuase otherwise, a read or write
    ' will destry the collection and/or leads into deadlocks ....
    '___________________________________________________________
    Private Sub InitVariableNames()
        mVarNames.Clear()

#If FLOWTYPE = 1 Then
        For Each var As Variable In mMain.Variables
            mVarNames.Add(var.Name)
        Next
#Else
        For Each prop As PropertyInfo In mMain.Variables.GetType().GetProperties()
            mVarNames.Add(prop.Name)
        Next
#End If

        Me.getVars()
    End Sub

    '___________________________________________________________
    ' Recreates the initial variable collection
    ' Important, this function must be calle ddirectly after the
    ' InitVariableNames
    '___________________________________________________________
#If FLOWTYPE = 1 Then         '====== ControlFlow
    Protected Function getVars() As Variables
#Else
#If VBC_VER = 8.0 Then
    Protected Function getVars() As IDTSVariables90     ' SQLServer 2005
#Else
    Protected Function getVars() As IDTSVariables100    ' SQLServer 2008
#End If
#End If
        Dim varName As String = ""
        Try
            Me.Unlock()
            For Each varName In mVarNames
                If varName.Length > 0 And mMain.VariableDispenser.Contains(varName) Then
                    Call mMain.VariableDispenser.LockForRead(varName)
                End If
            Next
            Call mMain.VariableDispenser.GetVariables(mVars)
        Catch ex As Exception
            Me.FireError(1, Me.ToString, String.Format("Error during lock of variable '{0}' : {1}", varName, ex.ToString))
        End Try
        Return mVars
    End Function

    Private Sub Unlock()
#If FLOWTYPE = 1 Then         '====== ControlFlow
        If mMain.Variables.Locked Then mMain.Variables.Unlock()
#End If
        If Not mVars Is Nothing Then
            If mVars.Locked Then mVars.Unlock()
        End If
    End Sub
End Class