## Software: ## (c) Vicenç Torra ## Current version: 20221201 ## ## Companion code of software for non-additive measures and integrals ## (these measures are also known as fuzzy measures, monotonic games, ## and include belief functions, k-additive measures, ## distorted probabilities) ## Software includes Choquet and Sugeno integrals, ## measure identification from examples, Shapley values, indices, etc. ## ## Companion code of software for data privacy, and papers on privacy. ## ## References: ## E. Turkarslan, V. Torra, Measure Identification for the Choquet integral: ## a Python module, I. J. of Computational Intelligence Systems, in press ## V. Torra, Y. Narukawa, Modeling Decisions, Springer, 2007. ## ## V. Torra, Guide to data privacy, Springer, 2022. from itertools import starmap from functools import reduce ## MATRICES ------------------------------------------------- ## def vec2mat (vec): """ Transforms a list into column matrix """ return list(map(lambda e:[e], vec)) ## Select a set of records, and the remaining set of records def matrixSelection (X, loIndices): notInLoIndices = [i for i in range(0,len(X)) if i not in loIndices] return (listSelection(X, loIndices), listSelection(X, notInLoIndices)) ## Select a set of columns records, and the remaining set of colums ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## matrixColumnSelection(mat,[1,2]) def matrixColumnSelection (X, loIndices): notInLoIndices = [i for i in range(0,len(X[0])) if i not in loIndices] # print(notInLoIndices) selectedX = [] unSelectedX = [] for i in range(0,len(X)): colsSelected, colsOther = matrixSelection(X[i], loIndices) selectedX.append(colsSelected) unSelectedX.append(colsOther) return(selectedX, unSelectedX) ## matrixProduct([[1,2],[2,3]],[[1],[2]]) # [[5], [8]] ## matrixProduct([[1],[2],[3]],[[1],[2],[3]]) # [[1], [2], [3]] -- WRONG DIMENSIONS ## matrixProduct([[1],[2],[3]],[[1,2,3]]) # [[1, 2, 3], [2, 4, 6], [3, 6, 9]] def matrixProduct (mat1, mat): mat2 = transpose(mat) return (list(map(lambda row: list(map(lambda col: vectorProduct (row, col), mat2)), mat1))) def matrixProductConstant (mat, a): """ matrixProductConstant([[1,2],[2,3]],4.1) """ return (list(map(lambda row: list(map(lambda col: col*a,row)), mat))) def matrixSum (mat1, mat2): """ matrixSum([[1,2,3],[4,5,6]],[[1,2,3],[4,5,6]]) """ return (list(map(lambda row1, row2: list(map(lambda e1, e2: e1+e2, row1, row2)), mat1, mat2))) def transpose(mat): """ Example: transpose([[1,2],[3,4]]) # [[1, 3], [2, 4]] transpose([[1,2],[3,4],[5,6]]) # [[1, 3, 5], [2, 4, 6]] """ nRows = len(mat) nCols = len(mat[0]) nMat = [] for i in range(0,nCols): nRow = [] for j in range(0,nRows): nRow.append(mat[j][i]) nMat.append(nRow) return(nMat) #def transpose(mat): # if (mat[0]==[]): # return [] # else: # firstRow = list(map(lambda row: row[0], mat)) # return ([firstRow] + # transpose(list(map(lambda row: row[1::], mat)))) ## Function: ## apply e^alpha to all elements e in the matrix ## Example: ## matrix2alpha([[1,2,3],[0,1,2],[-1,-2,-3]], 2) def matrix2alpha (mat, alpha): return list(map(lambda row:list(map(lambda e:e**alpha, row)), mat)) ## Function: ## Compute the means of all columns in the "matrix" ## Examples: ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## matrixColumnSums(mat) ## matOfMat = [mat,mat,mat] ## list(map(matrixColumnSums,matOfMat)) ## matrixColumnSums([]) def matrixColumnSums (mat): numRows = len(mat) res = [] if (numRows!=0): numCols = len(mat[0]) # print(numCols) for i in range(numCols): res.append(sum(map(lambda x:x[i], mat))) return(res) ## Function: ## Apply f column wise ## Examples ## matrixColumnFunction(mat,max) ## matrixColumnFunction(mat,min) def matrixColumnFunction (mat, f): numRows = len(mat) res = [] if (numRows!=0): numCols = len(mat[0]) # print(numCols) for i in range(numCols): res.append(f(list(map(lambda x:x[i], mat)))) return(res) ## Function: ## Compute the means of all columns in the "matrix" ## At least one row in the matrix ## Example: ## matrixColumnSumsReduce(mat) ## matrixColumnSumsReduce([]) def matrixColumnSumsReduce (mat): return(list(reduce (lambda acc,rowi: (map(lambda x,y: x+y, rowi, acc)), mat, [0]*len(mat[0])))) ## Function: ## Compute the min and maximum of all columns ## Example: ## matrixColumnMinMax(mat) def matrixColumnMinMax (mat): lMin = mat[0].copy() lMax = mat[0].copy() for i in range(0,len(mat)): lMin = list(map(lambda x,y: min(x,y), lMin, mat[i])) lMax = list(map(lambda x,y: max(x,y), mat[i], lMax)) return (lMin, lMax) ## Function: ## Compute the means of all columns in the "matrix" ## Examples: ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## matrixColumnMeans(mat) ## matOfMat = [mat,mat,mat] ## list(map(matrixColumnMeans,matOfMat)) ## matrixColumnMeans([]) def matrixColumnMeans (mat): numRows = len(mat) res = [] if (numRows!=0): numCols = len(mat[0]) # print(numCols) for i in range(numCols): res.append((sum(map(lambda x:x[i], mat)))/numRows) return(res) def matrixColumnMeansAbs (mat): numRows = len(mat) res = [] if (numRows!=0): numCols = len(mat[0]) # print(numCols) for i in range(numCols): res.append((sum(map(lambda x:abs(x[i]), mat)))/numRows) return(res) ## Function: ## compute the standard deviation of each column ## Examples: ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## matrixColumnSD(mat) def matrixColumnSD (mat): numRows = len(mat) numCols = len(mat[0]) colMeans = matrixColumnMeans (mat) res = [] for i in range(numCols): res.append(sum(map(lambda x: (x[i]-colMeans[i])**2.0, mat)) / (numRows-1)) return(res) ## Function: ## compute the variance of each column ## Examples: ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## matrixColumnVariance(mat) def matrixColumnVariance (mat): sd = matrixColumnSD (mat) return (list(map(lambda x:x*x, sd))) ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## matrixColumnSD(mat) ## Function: ## (x_ij - mean(x_.j))/sd(x_.j) ## Example: ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## normalizeMatrixByColumn(mat) ## normalizeMatrixByColumn(mat) def normalizeMatrixByColumn (mat): colMeans = matrixColumnMeans(mat) colSD = matrixColumnSD(mat) numRows = len(mat) res = [] for i in range(numRows): res.append(list(map(lambda xij,meanj,sdj: 0 if (sdj==0) else (xij-meanj)/sdj, mat[i], colMeans, colSD))) return(colMeans, colSD, res) def restoreNormalizationMatrixByColumn (colMeans, colSD, matNorm): numRows = len(matNorm) res = [] for i in range(numRows): res.append(list(map(lambda xNij, meanj, sdj: meanj if (sdj==0) else xNij*sdj + meanj, matNorm[i], colMeans, colSD))) return(res) ## Function: ## (x_ij - min(x_.j))/(max(x_.j) - min(x_.j)) ## Example: ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## lMin, lMax, matp = matrixScaleMinMax(mat) def matrixScaleMinMax (mat): lMin, lMax = matrixColumnMinMax (mat) res = [] for i in range(0,len(mat)): res.append(list(map(lambda ri, mn, mx: (ri-mn)/(mx-mn) if mn!=mx else 0.5, mat[i], lMin,lMax))) return (lMin, lMax,res) ## Function: ## min(x_.j) + (max(x_.j) - min(x_.j))*sx_ij ## Example: ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## lMin, lMax, matp = matrixScaleMinMax(mat) ## matrixUnScaleMinMax(lMin, lMax, matp) def matrixUnScaleMinMax (lMin, lMax, mat): res = [] for i in range(0,len(mat)): res.append(list(map(lambda sri, mn, mx: mn + (mx-mn)*sri if mn!=mx else mx, mat[i], lMin,lMax))) return res ## Function: ## return position of the farthest record to vect ## Example: ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## farthestRow(mat,[1.0,2.0,3.0]) def farthestRow (db, vect): selectedRow=0 dRow = fNorm2 (vect, db[selectedRow]) for i in range(1, len(db)): d = fNorm2 (vect, db[i]) if (d>dRow): # print("Row="+str(i)+" d="+str(d)) dRow=d selectedRow=i return(selectedRow) ## VECTORS and LISTS ------------------------------------------------- def listSelection (l, loIndices): """listSelection([0,1,2,3,4,5],[2,3,4]) """ return [l[i] for i in loIndices] ## Example: ## vectorSum([1,2,3],[4,5,6]) def vectorSum (l1, l2): return(list(map(lambda x,y:x+y, l1, l2))) def vectorProduct (vec1, vec2): """ Function: inner product, or simply product, of two vectors Example: product([1,2,3],[5,6,7]) """ return (sum(map(lambda x,y: x*y, vec1, vec2))) def vectorProductConstant (vec1, alpha): """ Function: inner product, or simply product, of two vectors Example: vectorProductConstant([1,2,3],2) """ return (list(map(lambda x: x*alpha, vec1))) ## Example: ## mean([1,2,3]) def mean (l1): return(sum(l1)/len(l1)) ## ## Function: ## normalize a vector so that \sum v_i = 1 ## Example: ## normalize([1,2,3,4]) == [0.1, 0.2, 0.3, 0.4] def normalize(v): s = sum(v) return(list(map(lambda x:x/s, v))) ## Function: ## Computes the square of the norm of the two vectors ||v1 - v2||^2 ## Example: ## fNorm2([1.0,2.0,3.0],[1.0,2.0,0.0]) def fNorm2 (v1, v2): return(sum(starmap(lambda e1, e2: (e1 - e2)*(e1 - e2), zip(v1,v2)))) def fNorm (v1, v2): return(math.sqrt(sum(starmap(lambda e1, e2: (e1 - e2)*(e1 - e2), zip(v1,v2))))) def vectNorm (v1): return(np.sqrt(sum(map(lambda e: e*e, v1)))) def vectCosineSim (v1, v2): """ vectCosineSim ([0.0,1.0,0.0],[0.0,1.0,0.0])""" return (vectorProduct(v1, v2) / (vectNorm(v1)*vectNorm(v2))) ## Function: ## Computes the maximum but there is a minimum value for this maximum ## Example: ## maxWithMinimum([-1,-2],0.0) def maxWithMinimum (lof, mn): if (lof==[]): mx=mn else: mx = max(lof) mx = max([mx, mn]) return(mx) ## Function: ## Select lofe[i] if f(lofv[i)) is true ## Example: ## mat = [[1.0,2.0,3.0],[2.0,2.0,4.0],[3.0,2.0,5.0],[2.0,2.0,4.0]] ## selectFv(mat, [0,1,1,1], lambda x:x==0) ## selectFv(mat, [0,1,1,1], lambda x:x==1) ## selectFv(mat, [0,1,1,1], lambda x:x==1) def selectFv (lofe, lofv, f): res = [] for i in range(0, len(lofe)): if (f(lofv[i])): res.append(lofe[i]) return(res) ## From: ## https://stackoverflow.com/questions/952914/how-to-make-a-flat-list-out-of-a-list-of-lists ## Example: ## flatten([[[1,1],[2,2]],[[3,3,3]],[[4,4,4],[4,4,4]]]) ## --> [[1, 1], [2, 2], [3, 3, 3], [4, 4, 4], [4, 4, 4]] ## flatten([[1,2,3],[4,4,4],[3],[[[100]]]]) ## --> [1, 2, 3, 4, 4, 4, 3, [[100]]] def flatten(t): return [item for sublist in t for item in sublist] ## Function: ## Input: pairs (n_i, v_i) ## Output: a list with n_i copies of each v_i ## Example: ## nCopies([(1,[1,1,1]), (2,[4,4,4]), (5,[8,9])]) ## --> [[1, 1, 1], [4, 4, 4], [4, 4, 4], [8, 9], [8, 9], [8, 9], [8, 9], [8, 9]] ## IMPORTANT NOTE: the elements in v are shared !! ## nCopies([(1,[1,1,1]), (2,[4,4,4]), (5,[8,9])]) ## a=[1,2,3] ## b=nCopies([(1,a),(2,a),(3,a)]) ## a[2]=1000 ## b def nCopies(listOfPairs): return(list(flatten(starmap(lambda n, v: [v]*n, listOfPairs)))) def sortWithOneListIncreasing (toSort, theList): """ Function: sort a list, and apply the same permutation to the second. (from lowest to largest) Example: sortWithOneListIncreasing ([1,4,2,5,3],["first", "2nd", "3rd", "4th","last"]) """ for i in range(0,len(toSort)): for j in range(i+1, len(toSort)): if (toSort[j]toSort[i]): swap = toSort[j] toSort[j]=toSort[i] toSort[i]=swap swap = theList[j] theList[j] = theList[i] theList[i] = swap return((toSort,theList)) def remove (l, e): """Function: Delete, remove an element from a list. Example: remove([1,2,3,4,5],1)""" if l==[]: return (l) elif e==l[0]: return (l[1:]) else: return [l[0]]+remove(l[1:],e) ## --------------------------------------------------------------- ## Files ## --------------------------------------------------------------- ## import csv ## Function: ## read a file. First line is header ## Example: ## fheader, fRows = readNumericalFileWithHeader("Concrete_DataVarsV1V7.csv") ## print(fRows) ## print(fheader) def readNumericalFileWithHeader(nameFile): file = open(nameFile) csvreader = csv.reader(file) header = next(csvreader) ## print(header) rows = [] for row in csvreader: nrow = list(map(lambda e:(float(e)),row)) rows.append(nrow) file.close() return(header, rows) def readNumericalFileWithComments(nameFile): file = open(nameFile) csvreader = csv.reader(file) ## print(header) rows = [] for row in csvreader: if row[0][0]!='#': nrow = list(map(lambda e:(float(e)),row)) rows.append(nrow) file.close() return(rows) # db = readNumericalFileWithComments("abalone.data.numeric.csv") ## --------------------------------------------------------------- ## OTHER AUXILIARY FUNCTIONS ## --------------------------------------------------------------- ## def printAndReturn(e): print(e) return(e) ### ### FUNCTIONS ON SETS ----------------- ### def equalSets(mySet1, mySet2, n): """ Function: Check if the two sets are equal Example: equalSets([0,1,1,0],[0,1,1,0],4) """ return(mySet1==mySet2) def setIntersection (mySet1, mySet2, n): """ Function: Compute the intersection of two sets. Example: setIntersection([0,1,1],[1,1,0],3) """ return (list(map(lambda x,y:min(x,y), mySet1, mySet2))) def setUnion (mySet1, mySet2, n): """ Function: Compute the intersection of two sets. Example: setUnion([0,1,1],[1,1,0],3) """ return (list(map(lambda x,y:max(x,y), mySet1, mySet2))) def subseteq(mySet1, mySet2, n): """ Function: mySet1 \subseteq mySet2? Example: subseteq([0,1,1,0],[0,1,1,1],4) # == True subseteq([1,1,1,0],[0,1,1,1],4) # == False """ subset = True i = 0 while (subset and (i < n)): if mySet1[i]==1: subset = (mySet2[i]==1) i=i+1 return(subset) def belongSet(i, mySet, n): """ Function: Check if an element is in the set Example: belongSet(2, [0,1,1,0],4) belongSet(0, [0,1,1,0],4) """ return(mySet[i]==1) def addElement2Set(i, mySet, n): """ Function: add an element to the set Example: addElement2Set(2, [0,1,1,0],4) addElement2Set(0, [0,1,1,0],4) """ mySet[i]=1 return(mySet) def removeElement2Set(i, mySet, n): """ Function: remove an element from a set Example: removeElement2Set(2, [0,1,1,0],4) removeElement2Set(0, [0,1,1,0],4) """ mySet[i]=0 return(mySet) def firstSet(n): """ Function: returns first set == [0,...,0] Example: firstSet(4) # == [0,0,0,0] """ return([0]*n) def lastSet(mySet): """ Function: Last possible set? Example: lastSet([1,1,1,1]) # == True lastSet([1,0,1,1]) # == False """ return(not(0==min(mySet))) def notLastSet(mySet): """ Function: still other sets? Example: notLastSet([1,1,1,1]) # == False notLastSet([1,0,1,1]) # == True """ return(0==min(mySet)) def nextSet(mySet, n): """ Function: Given a set, next one (assume not last one!) Example: nextSet([0,0,1,0],4) #==[0,0,1,1] """ i=n-1 if (mySet[i]==0): mySet[i]=1 else: while(mySet[i]==1): mySet[i]=0 i=i-1 mySet[i]=1 return(mySet) def nElemsSet(mySet, n): """ Function: count number of elements in the set Example: nElemsSet([1,0,1,1],4) """ return(sum(mySet)) ## In Scala vect2int def set2Int (vec): """ Function: given a set [0,1,1,0] returns the integer that represents Example: set2Int([1,1,1,1]) # 15 set2Int([1,1,0,1]) # 13 """ return(sum(map(lambda x,i:x*(2**i), vec, list(range(len(vec)-1, -1, -1))))) def setSet2Index (mySet,n, nameElements=None): """ Function: Given a set [0,1,1,0] returns the list of elements in the set Examples: setSet2Index ([0,1,1,0,1],5) setSet2Index ([0,1,1,0,1],5,["a","b","c","d","e"]) """ if nameElements == None: nameElements = range(0,n) return [nameElements[i] for i in range(0,n) if mySet[i]==1] def setIndex2Set (vec,n): """ Function: given a set of indices [1,4] returns a set (i.e., a list) Example: setIndex2Set([0],6) # == [1, 0, 0, 0, 0, 0] setIndex2Set([1],6) # == [0, 1, 0, 0, 0, 0] setIndex2Set([0,1],6) # == [1, 1, 0, 0, 0, 0] setIndex2Set([5],6) # == [0, 0, 0, 0, 0, 1] """ mySet = [0]*n for i in vec: mySet[i]=1 return(mySet) def setIndex2Int (vec,n): """ Function: given a set of indices [1,4] returns the integer that represents Example: setIndex2Int([0],6) # == 32 setIndex2Int([1],6) # == 16 setIndex2Int([0,1],6)# == 48 setIndex2Int([0,3],6)# == 36 setIndex2Int([5],6)# == 1 """ return(set2Int(setIndex2Set (vec,n))) def setAllElems2ElemsInSet (mySet, allElems): """ Example: setAllElems2ElemsInSet([0,1,0,1],[1,0,-45,2],4) """ return([x[0] for x in zip(allElems, mySet) if x[1]==1]) def permutations (l): """Function: Build al permutations for the elements in l. Example: permutations([1,2,3,4,5])""" if l==[]: return [] elif len(l)==1: return [[l[0]]] else: return flatten(list(map(lambda e: list(map(lambda perm: [e]+perm, permutations(remove(l,e)))),l)))