From ffaf4aed3498b0eed5155ab5a92fe791cd174ab9 Mon Sep 17 00:00:00 2001 From: Matt Soucy Date: Thu, 25 Jun 2020 21:23:03 -0400 Subject: [PATCH] Starting summary model This is rather large... --- src/main/kotlin/me/msoucy/gbat/Models.kt | 319 +++++++++++++++++++++-- 1 file changed, 292 insertions(+), 27 deletions(-) diff --git a/src/main/kotlin/me/msoucy/gbat/Models.kt b/src/main/kotlin/me/msoucy/gbat/Models.kt index 03785e1..8255314 100644 --- a/src/main/kotlin/me/msoucy/gbat/Models.kt +++ b/src/main/kotlin/me/msoucy/gbat/Models.kt @@ -1,7 +1,9 @@ package me.msoucy.gbat import java.io.File +import java.nio.file.Paths import kotlin.io.forEachLine +import org.jetbrains.exposed.dao.id.IntIdTable import org.jetbrains.exposed.sql.* import org.jetbrains.exposed.sql.transactions.transaction @@ -97,21 +99,20 @@ class LineModel() { } -class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { +class KnowledgeModel(val db : Database, val constant : Double, val riskModel : RiskModel) { + class KnowledgeAcct(var knowledgeAcctId : Int, var authors : List, var authorsStr : String) object AuthorsTable : Table("authors") { - val id = integer("id") - val author = text("author") - val idx = integer("idx").uniqueIndex() + val id = integer("authorid") + val author = text("author").uniqueIndex("authors_idx") override val primaryKey = PrimaryKey(id) } object KnowledgeAcctsTable : Table("knowledgeaccts") { - val id = integer("id") - val authors = text("authors") - val idx = integer("idx").uniqueIndex() + val id = integer("knowledgeacctid") + val authors = text("authors").uniqueIndex("knowledgeacctsauthors_idx") override val primaryKey = PrimaryKey(id) } object KnowledgeAuthorsTable : Table("knowedgeaccts_authors") { @@ -125,6 +126,10 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { val knowledge = double("knowledge") } + init { + createTables() + } + val SAFE_AUTHOR_ID = 1 val SAFE_KNOWLEDGE_ACCT_ID = 1 val KNOWLEDGE_PER_LINE_ADDED = 1000.0 @@ -160,7 +165,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { adjustKnowledge(knowledgeAcctId, lineNum, KNOWLEDGE_PER_LINE_ADDED) } - fun knowledgeSummary(lineNum : Int) = transaction { + fun knowledgeSummary(lineNum : Int) = transaction(db) { LineKnowledge.select { LineKnowledge.linenum eq lineNum }.map { @@ -171,7 +176,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { }.copyOf() } - private fun bumpAllLinesFrom(lineNum : Int, adjustment : Int) = transaction { + private fun bumpAllLinesFrom(lineNum : Int, adjustment : Int) = transaction(db) { LineKnowledge.update({LineKnowledge.linenum greater lineNum}) { with(SqlExpressionBuilder) { it[LineKnowledge.linenum] = LineKnowledge.linenum + adjustment @@ -179,7 +184,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { } } - private fun getKnowledgeAcct(knowledgeAcctId : Int) = transaction { + private fun getKnowledgeAcct(knowledgeAcctId : Int) = transaction(db) { KnowledgeAcctsTable.select { KnowledgeAcctsTable.id eq knowledgeAcctId }.map { @@ -191,7 +196,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { }.first() } - private fun destroyLineKnowledge(knowledgeId : Int, lineNum : Int) = transaction { + private fun destroyLineKnowledge(knowledgeId : Int, lineNum : Int) = transaction(db) { LineKnowledge.deleteWhere { (LineKnowledge.knowledgeacctid eq knowledgeId) and (LineKnowledge.linenum eq lineNum) @@ -226,7 +231,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { } } - private fun knowledgeInAcct(knowledgeAcctId : Int, lineNum : Int) = transaction { + private fun knowledgeInAcct(knowledgeAcctId : Int, lineNum : Int) = transaction(db) { LineKnowledge.select { (LineKnowledge.knowledgeacctid eq knowledgeAcctId) and (LineKnowledge.linenum eq lineNum) @@ -235,7 +240,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { }.first() } - private fun nonSafeAcctsWithKnowledgeOf(lineNum : Int) = transaction { + private fun nonSafeAcctsWithKnowledgeOf(lineNum : Int) = transaction(db) { LineKnowledge.select { (LineKnowledge.linenum eq lineNum) and (LineKnowledge.knowledgeacctid neq SAFE_KNOWLEDGE_ACCT_ID) @@ -244,7 +249,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { } } - private fun allAcctsWithKnowledgeOf(lineNum : Int) = transaction { + private fun allAcctsWithKnowledgeOf(lineNum : Int) = transaction(db) { LineKnowledge.select { LineKnowledge.linenum eq lineNum }.map { @@ -252,7 +257,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { } } - private fun adjustKnowledge(knowledgeAcctId : Int, lineNum : Int, adjustment : Double) = transaction { + private fun adjustKnowledge(knowledgeAcctId : Int, lineNum : Int, adjustment : Double) = transaction(db) { val lineExists = LineKnowledge.select { (LineKnowledge.knowledgeacctid eq knowledgeAcctId) and (LineKnowledge.linenum eq lineNum) @@ -274,7 +279,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { } } - private fun lookupOrCreateKnowledgeAcct(authors : List) = transaction { + private fun lookupOrCreateKnowledgeAcct(authors : List) = transaction(db) { val authorStr = authors.sorted().joinToString("\n") var newId = -1 KnowledgeAcctsTable.select { @@ -293,8 +298,7 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { it[KnowledgeAcctsTable.id] }.first() - authors.map(::lookupOrCreateAuthor). - forEach { authorId -> + authors.map(::lookupOrCreateAuthor).forEach { authorId -> KnowledgeAuthorsTable.insert { it[KnowledgeAuthorsTable.knowledgeacctid] = newId it[KnowledgeAuthorsTable.authorid] = authorId @@ -304,27 +308,26 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { newId } - private fun lookupOrCreateAuthor(authorName : String) = transaction { + private fun lookupOrCreateAuthor(authorName : String) = transaction(db) { AuthorsTable.insertIgnore { it[author] = authorName } AuthorsTable.select { AuthorsTable.author eq authorName - }.fetchSize(1). - map { + }.fetchSize(1).map { it[AuthorsTable.id] }.first() } - private fun totalLineKnowledge(linenum : Int) = transaction { + private fun totalLineKnowledge(linenum : Int) = transaction(db) { LineKnowledge.select { LineKnowledge.linenum eq linenum - }.fetchSize(1). - map { it[LineKnowledge.knowledge] }. - sum() + }.fetchSize(1).map { + it[LineKnowledge.knowledge] + }.first() } - private fun createTables() = transaction { + private fun createTables() = transaction(db) { SchemaUtils.createMissingTablesAndColumns(AuthorsTable, KnowledgeAcctsTable, KnowledgeAuthorsTable, LineKnowledge) AuthorsTable.insertIgnore { it[id] = 1 @@ -334,9 +337,271 @@ class KnowledgeModel(val constant : Double, val riskModel : RiskModel) { it[id] = 1 it[authors] = "" } - KnowledgeAuthorsTable.insertIgnore { + KnowledgeAuthorsTable.insertIgnore { it[knowledgeacctid] = SAFE_KNOWLEDGE_ACCT_ID it[authorid] = SAFE_AUTHOR_ID } } +} + +class SummaryModel(val db : Database) { + + object ProjectTable : IntIdTable("projects", "projectid") { + val project = text("project").uniqueIndex("project_idx") + } + object DirsTable : IntIdTable("dirs", "dirid") { + val dir = text("dir") + val parentdirid = integer("parentdirid") + val projectid = integer("projectid") + val dirsproj_idx = uniqueIndex("dirsproj_idx", dir, parentdirid, projectid) + } + object FilesTable : IntIdTable("files", "fileid") { + val fname = text("fname") + val dirid = integer("dirid").index("filesdir_idx") + } + object LinesTable : IntIdTable("lines", "lineid") { + val line = text("line") + val fileid = integer("fileid").index("linesfile_idx") + val linenum = integer("linenum") + val linesnumfile_idx = uniqueIndex("linesnumfile_idx", fileid, linenum) + } + object AuthorsTable : IntIdTable("authors", "authorid") { + val author = text("author").uniqueIndex("authorstrs_idx") + } + object AuthorsGroupsTable : IntIdTable("authorgroups", "authorgroupid") { + val authors = text("authorsstr").uniqueIndex("authorgroupsstrs_idx") + } + object AuthorsAuthorGroupsTable : Table("authors_authorgroups") { + val authorid = integer("authorid") + val groupid = integer("authorgroupid") + override val primaryKey = PrimaryKey(authorid, groupid) + } + object AllocationsTable : IntIdTable("allocations", "allocationid") { + val knowledge = double("knowledge") + val risk = double("risk") + val orphaned = double("orphaned") + val lineid = integer("lineid").index("linealloc_idx") + val authorgroupid = integer("authorgroupid") + } + + val GIT_BY_A_BUS_BELOW_THRESHOLD = "Git by a Bus Safe Author" + + init { + createTables() + } + + data class Statistics(var totKnowledge : Double = 0.0, + var totRisk : Double = 0.0, + var totOrphaned : Double = 0.0) { + constructor(row : ResultRow) : + this(row[AllocationsTable.knowledge.sum()] ?: 0.0, + row[AllocationsTable.risk.sum()] ?: 0.0, + row[AllocationsTable.orphaned.sum()] ?: 0.0) {} + } + data class AuthorRisk(var stats : Statistics = Statistics()) + data class LineDict(var stats : Statistics = Statistics(), var authorRisks : MutableMap = mutableMapOf()) + class FileTree(var name : String = "", + var stats : Statistics = Statistics(), + var authorRisks : MutableMap = mutableMapOf(), + var lines : MutableList = mutableListOf()) + + fun fileSummary(fileId : Int) = transaction(db) { + var fileTree = FileTree() + val joinA = Join(LinesTable, AllocationsTable, + JoinType.LEFT, + LinesTable.id, AllocationsTable.lineid) + val joinB = Join(joinA, AuthorsGroupsTable, + JoinType.LEFT, + AuthorsGroupsTable.id, AllocationsTable.authorgroupid + ) + joinB.select { + LinesTable.fileid eq fileId + }.groupBy(AuthorsGroupsTable.id).forEach { row -> + val authors = row[AuthorsGroupsTable.authors] + fileTree.authorRisks[authors] = + AuthorRisk(Statistics(row)) + } + Join(joinA, FilesTable, + JoinType.LEFT, + LinesTable.fileid, FilesTable.id + ).select { + LinesTable.fileid eq fileId + }.fetchSize(1).forEach { row -> + fileTree.stats = Statistics(row) + } + + fileTree.name = FilesTable.select { + FilesTable.id eq fileId + }.map { it[FilesTable.fname] }.first() + + LinesTable.select { + LinesTable.fileid eq fileId + }.map { + it[LinesTable.id].value + }.forEach { lineId -> + val lineDict = LineDict() + joinB.select { + LinesTable.id eq lineId + }.groupBy(AuthorsGroupsTable.id).forEach { lineRow -> + lineDict.authorRisks[lineRow[AuthorsGroupsTable.authors]] = AuthorRisk(Statistics(lineRow)) + } + + joinA.select { + LinesTable.id eq lineId + }.fetchSize(1).forEach { + lineDict.stats = Statistics(it) + } + fileTree.lines.add(lineDict) + } + fileTree + } + + fun fileLines(fileId : Int) : List = transaction(db) { + LinesTable.select { + LinesTable.fileid eq fileId + }.orderBy(LinesTable.linenum).map { + it[LinesTable.line] + } + } + + private fun Double?.zeroIfNone() = this ?: 0.0 + + private fun reconsDir(dirId : Int) = transaction(db) { + val segs = mutableListOf() + var newDirId = dirId + while(newDirId != 0) { + DirsTable.select { + DirsTable.id eq newDirId + }.forEach { + segs.add(it[DirsTable.dir]) + newDirId = it[DirsTable.parentdirid] + } + } + Paths.get(segs.reversed().joinToString("/")).normalize() + } + + private fun safeAuthorName(author : String?) = author ?: GIT_BY_A_BUS_BELOW_THRESHOLD + + private fun createAllocation(knowledge : Double, risk : Double, orphaned : Double, authorGroupId : Int, lineId : Int) = transaction(db) { + AllocationsTable.insert { + it[AllocationsTable.knowledge] = knowledge + it[AllocationsTable.risk] = risk + it[AllocationsTable.orphaned] = orphaned + it[AllocationsTable.lineid] = lineId + it[AllocationsTable.authorgroupid] = authorGroupId + } + } + + private fun findOrCreateAuthorGroup(authors : List) : Int = transaction(db) { + val authorsstr = authors.joinToString("\n") + var authorGroupId = AuthorsGroupsTable.select { + AuthorsGroupsTable.authors eq authorsstr + }.map { + it[AuthorsGroupsTable.id].value + }.firstOrNull() + + if (authorGroupId == null) { + authorGroupId = AuthorsGroupsTable.insertAndGetId { + it[AuthorsGroupsTable.authors] = authorsstr + }.value + authors.forEach { + val authorId = findOrCreateAuthor(it) + AuthorsAuthorGroupsTable.insert { + it[AuthorsAuthorGroupsTable.authorid] = authorId + it[AuthorsAuthorGroupsTable.groupid] = authorGroupId + } + } + } + authorGroupId + } + + private fun findOrCreateAuthor(author : String) : Int = transaction(db) { + AuthorsTable.insertIgnore { + it[AuthorsTable.author] = author + } + ProjectTable.select { + AuthorsTable.author eq author + }.map { + it[AuthorsTable.id].value + }.first() + } + + private fun createLine(line : String, lineNum : Int, fileId : Int) = transaction(db) { + LinesTable.insertAndGetId { + it[LinesTable.line] = line + it[LinesTable.linenum] = lineNum + it[LinesTable.fileid] = fileId + }.value + } + + private fun createFile(fname : String, parentDirId : Int) = transaction(db) { + FilesTable.insertAndGetId { + it[FilesTable.fname] = fname + it[FilesTable.dirid] = parentDirId + }.value + } + + private fun findOrCreateProject(project : String) : Int = transaction(db) { + ProjectTable.insertIgnore { + it[ProjectTable.project] = project + } + ProjectTable.select { + ProjectTable.project eq project + }.map { + it[ProjectTable.id].value + }.first() + } + + private fun findOrCreateDir(dirname : String, projectId : Int, parentDirId : Int) : Int = transaction(db) { + DirsTable.insertIgnoreAndGetId { + it[dir] = dirname + it[parentdirid] = parentDirId + it[projectid] = projectId + } + DirsTable.select { + DirsTable.dir eq dirname + DirsTable.parentdirid eq parentDirId + DirsTable.projectid eq projectId + }.map { + it[DirsTable.id] + }.first().value + } + + private fun splitAllDirs(dirname : File) = dirname.toPath().iterator().asSequence().toList() + + private fun adjustFname(repoRoot : File, projectRoot : File, fname : File) : File { + var rootDiff = projectRoot.relativeTo(repoRoot) + return if(rootDiff.toString().length != 0) { + fname.relativeTo(rootDiff) + } else { + fname + } + } + + private fun reconsDirs(dirId : Int) = transaction(db) { + val dirs = mutableListOf() + var parentDirId = dirId + while(parentDirId != 0) { + DirsTable.select { + DirsTable.id eq parentDirId + }.fetchSize(1). + forEach { + dirs.add(it[DirsTable.dir]) + parentDirId = it[DirsTable.parentdirid] + } + } + dirs.reversed() + } + + private fun createTables() = transaction(db) { + SchemaUtils.createMissingTablesAndColumns( + ProjectTable, + FilesTable, + LinesTable, + AuthorsTable, + AuthorsGroupsTable, + AuthorsAuthorGroupsTable, + AllocationsTable + ) + } } \ No newline at end of file