Skip to content

Commit 56a5c78

Browse files
igfooigfoo
authored
Merge pull request #14220 from igfoo/igfoo/light_comments
Kotlin: Extract LighterAST comments as well as PSI comments
2 parents 06cb277 + 56b9fa8 commit 56a5c78

File tree

5 files changed

+302
-164
lines changed

5 files changed

+302
-164
lines changed

‎java/kotlin-extractor/src/main/kotlin/KotlinFileExtractor.kt‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package com.github.codeql
22

3-
import com.github.codeql.comments.CommentExtractor
3+
import com.github.codeql.comments.CommentExtractorPSI
4+
import com.github.codeql.comments.CommentExtractorLighterAST
45
import com.github.codeql.utils.*
56
import com.github.codeql.utils.versions.*
67
import com.semmle.extractor.java.OdasaOutput
@@ -127,7 +128,15 @@ open class KotlinFileExtractor(
127128
}
128129
}
129130
extractStaticInitializer(file, { extractFileClass(file) })
130-
CommentExtractor(this, file, tw.fileId).extract()
131+
val psiCommentsExtracted = CommentExtractorPSI(this, file, tw.fileId).extract()
132+
val lighterAstCommentsExtracted = CommentExtractorLighterAST(this, file, tw.fileId).extract()
133+
if (psiCommentsExtracted == lighterAstCommentsExtracted) {
134+
if (psiCommentsExtracted) {
135+
logger.warnElement("Found both PSI and LightAST comments in ${file.path}.", file)
136+
} else {
137+
logger.warnElement("Comments could not be processed in ${file.path}.", file)
138+
}
139+
}
131140

132141
if (!declarationStack.isEmpty()) {
133142
logger.errorElement("Declaration stack is not empty after processing the file", file)

‎java/kotlin-extractor/src/main/kotlin/comments/CommentExtractor.kt‎

Lines changed: 57 additions & 162 deletions
Original file line numberDiff line numberDiff line change
@@ -1,187 +1,82 @@
11
package com.github.codeql.comments
22

33
import com.github.codeql.*
4-
import com.github.codeql.utils.IrVisitorLookup
54
import com.github.codeql.utils.isLocalFunction
6-
import com.github.codeql.utils.Psi2IrFacade
7-
import com.github.codeql.utils.versions.getPsi2Ir
8-
import com.intellij.psi.PsiComment
9-
import com.intellij.psi.PsiElement
10-
import org.jetbrains.kotlin.config.KotlinCompilerVersion
115
import org.jetbrains.kotlin.ir.IrElement
126
import org.jetbrains.kotlin.ir.declarations.*
137
import org.jetbrains.kotlin.ir.expressions.IrBody
148
import org.jetbrains.kotlin.ir.expressions.IrExpression
15-
import org.jetbrains.kotlin.ir.util.isAnonymousObject
169
import org.jetbrains.kotlin.ir.util.parentClassOrNull
17-
import org.jetbrains.kotlin.kdoc.psi.api.KDoc
18-
import org.jetbrains.kotlin.lexer.KtTokens
19-
import org.jetbrains.kotlin.psi.KtVisitor
20-
import org.jetbrains.kotlin.psi.psiUtil.endOffset
21-
import org.jetbrains.kotlin.psi.psiUtil.startOffset
2210

23-
class CommentExtractor(private val fileExtractor: KotlinFileExtractor, private val file: IrFile, private val fileLabel: Label<out DbFile>) {
24-
private val tw = fileExtractor.tw
25-
private val logger = fileExtractor.logger
11+
open class CommentExtractor(protected val fileExtractor: KotlinFileExtractor, protected val file: IrFile, protected val fileLabel: Label<out DbFile>) {
12+
protected val tw = fileExtractor.tw
13+
protected val logger = fileExtractor.logger
2614

27-
fun extract() {
28-
val psi2Ir = getPsi2Ir()
29-
if (psi2Ir == null) {
30-
logger.warn("Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
31-
return
15+
protected fun getLabel(element: IrElement): Label<out DbTop>? {
16+
if (element == file)
17+
return fileLabel
18+
19+
if (element is IrValueParameter && element.index == -1) {
20+
// Don't attribute comments to the implicit `this` parameter of a function.
21+
return null
22+
}
23+
24+
val label: String
25+
val existingLabel = if (element is IrVariable) {
26+
// local variables are not named globally, so we need to get them from the variable label cache
27+
label = "variable ${element.name.asString()}"
28+
tw.getExistingVariableLabelFor(element)
29+
} else if (element is IrFunction && element.isLocalFunction()) {
30+
// local functions are not named globally, so we need to get them from the local function label cache
31+
label = "local function ${element.name.asString()}"
32+
fileExtractor.getExistingLocallyVisibleFunctionLabel(element)
33+
} else {
34+
label = getLabelForNamedElement(element) ?: return null
35+
tw.getExistingLabelFor<DbTop>(label)
3236
}
33-
val ktFile = psi2Ir.getKtFile(file)
34-
if (ktFile == null) {
35-
logger.warn("Comments are not being processed in ${file.path}.")
36-
return
37+
if (existingLabel == null) {
38+
logger.warn("Couldn't get existing label for $label")
39+
return null
3740
}
38-
val commentVisitor = mkCommentVisitor(psi2Ir)
39-
ktFile.accept(commentVisitor)
41+
return existingLabel
4042
}
4143

42-
private fun mkCommentVisitor(psi2Ir: Psi2IrFacade): KtVisitor<Unit, Unit> =
43-
object : KtVisitor<Unit, Unit>() {
44-
override fun visitElement(element: PsiElement) {
45-
element.acceptChildren(this)
46-
47-
// Slightly hacky, but `visitComment` doesn't seem to visit comments with `tokenType` `KtTokens.DOC_COMMENT`
48-
if (element is PsiComment){
49-
visitCommentElement(element)
50-
}
51-
}
52-
53-
private fun visitCommentElement(comment: PsiComment) {
54-
val type: CommentType = when (comment.tokenType) {
55-
KtTokens.EOL_COMMENT -> {
56-
CommentType.SingleLine
57-
}
58-
KtTokens.BLOCK_COMMENT -> {
59-
CommentType.Block
60-
}
61-
KtTokens.DOC_COMMENT -> {
62-
CommentType.Doc
63-
}
64-
else -> {
65-
logger.warn("Unhandled comment token type: ${comment.tokenType}")
66-
return
67-
}
68-
}
69-
70-
val commentLabel = tw.getFreshIdLabel<DbKtcomment>()
71-
tw.writeKtComments(commentLabel, type.value, comment.text)
72-
val locId = tw.getLocation(comment.startOffset, comment.endOffset)
73-
tw.writeHasLocation(commentLabel, locId)
74-
75-
if (comment.tokenType != KtTokens.DOC_COMMENT) {
76-
return
77-
}
78-
79-
if (comment !is KDoc) {
80-
logger.warn("Unexpected comment type with DocComment token type.")
81-
return
82-
}
83-
84-
for (sec in comment.getAllSections()) {
85-
val commentSectionLabel = tw.getFreshIdLabel<DbKtcommentsection>()
86-
tw.writeKtCommentSections(commentSectionLabel, commentLabel, sec.getContent())
87-
val name = sec.name
88-
if (name != null) {
89-
tw.writeKtCommentSectionNames(commentSectionLabel, name)
90-
}
91-
val subjectName = sec.getSubjectName()
92-
if (subjectName != null) {
93-
tw.writeKtCommentSectionSubjectNames(commentSectionLabel, subjectName)
94-
}
95-
}
96-
97-
// Only storing the owner of doc comments:
98-
val ownerPsi = getKDocOwner(comment) ?: return
99-
100-
val owners = mutableListOf<IrElement>()
101-
file.accept(IrVisitorLookup(psi2Ir, ownerPsi, file), owners)
102-
103-
for (ownerIr in owners) {
104-
val ownerLabel = getLabel(ownerIr)
105-
if (ownerLabel != null) {
106-
tw.writeKtCommentOwners(commentLabel, ownerLabel)
107-
}
108-
}
109-
}
110-
111-
private fun getKDocOwner(comment: KDoc) : PsiElement? {
112-
val owner = comment.owner
113-
if (owner == null) {
114-
logger.warn("Couldn't get owner of KDoc. The comment is extracted without an owner.")
115-
}
116-
return owner
117-
}
118-
119-
private fun getLabel(element: IrElement): Label<out DbTop>? {
120-
if (element == file)
121-
return fileLabel
122-
123-
if (element is IrValueParameter && element.index == -1) {
124-
// Don't attribute comments to the implicit `this` parameter of a function.
125-
return null
126-
}
127-
128-
val label: String
129-
val existingLabel = if (element is IrVariable) {
130-
// local variables are not named globally, so we need to get them from the variable label cache
131-
label = "variable ${element.name.asString()}"
132-
tw.getExistingVariableLabelFor(element)
133-
} else if (element is IrFunction && element.isLocalFunction()) {
134-
// local functions are not named globally, so we need to get them from the local function label cache
135-
label = "local function ${element.name.asString()}"
136-
fileExtractor.getExistingLocallyVisibleFunctionLabel(element)
44+
private fun getLabelForNamedElement(element: IrElement) : String? {
45+
when (element) {
46+
is IrClass -> return fileExtractor.getClassLabel(element, listOf()).classLabel
47+
is IrTypeParameter -> return fileExtractor.getTypeParameterLabel(element)
48+
is IrFunction -> {
49+
return if (element.isLocalFunction()) {
50+
null
13751
} else {
138-
label = getLabelForNamedElement(element) ?: return null
139-
tw.getExistingLabelFor<DbTop>(label)
52+
fileExtractor.getFunctionLabel(element, null)
14053
}
141-
if (existingLabel == null) {
142-
logger.warn("Couldn't get existing label for $label")
54+
}
55+
is IrValueParameter -> return fileExtractor.getValueParameterLabel(element, null)
56+
is IrProperty -> return fileExtractor.getPropertyLabel(element)
57+
is IrField -> return fileExtractor.getFieldLabel(element)
58+
is IrEnumEntry -> return fileExtractor.getEnumEntryLabel(element)
59+
is IrTypeAlias -> return fileExtractor.getTypeAliasLabel(element)
60+
61+
is IrAnonymousInitializer -> {
62+
val parentClass = element.parentClassOrNull
63+
if (parentClass == null) {
64+
logger.warnElement("Parent of anonymous initializer is not a class", element)
14365
return null
14466
}
145-
return existingLabel
67+
// Assign the comment to the class. The content of the `init` blocks might be extracted in multiple constructors.
68+
return getLabelForNamedElement(parentClass)
14669
}
14770

148-
private fun getLabelForNamedElement(element: IrElement) : String? {
149-
when (element) {
150-
is IrClass -> return fileExtractor.getClassLabel(element, listOf()).classLabel
151-
is IrTypeParameter -> return fileExtractor.getTypeParameterLabel(element)
152-
is IrFunction -> {
153-
return if (element.isLocalFunction()) {
154-
null
155-
} else {
156-
fileExtractor.getFunctionLabel(element, null)
157-
}
158-
}
159-
is IrValueParameter -> return fileExtractor.getValueParameterLabel(element, null)
160-
is IrProperty -> return fileExtractor.getPropertyLabel(element)
161-
is IrField -> return fileExtractor.getFieldLabel(element)
162-
is IrEnumEntry -> return fileExtractor.getEnumEntryLabel(element)
163-
is IrTypeAlias -> return fileExtractor.getTypeAliasLabel(element)
71+
// Fresh entities, not named elements:
72+
is IrBody -> return null
73+
is IrExpression -> return null
16474

165-
is IrAnonymousInitializer -> {
166-
val parentClass = element.parentClassOrNull
167-
if (parentClass == null) {
168-
logger.warnElement("Parent of anonymous initializer is not a class", element)
169-
return null
170-
}
171-
// Assign the comment to the class. The content of the `init` blocks might be extracted in multiple constructors.
172-
return getLabelForNamedElement(parentClass)
173-
}
174-
175-
// Fresh entities, not named elements:
176-
is IrBody -> return null
177-
is IrExpression -> return null
178-
179-
// todo add others:
180-
else -> {
181-
logger.warnElement("Unhandled element type found during comment extraction: ${element::class}", element)
182-
return null
183-
}
184-
}
75+
// todo add others:
76+
else -> {
77+
logger.warnElement("Unhandled element type found during comment extraction: ${element::class}", element)
78+
return null
79+
}
18580
}
18681
}
18782
}

‎java/kotlin-extractor/src/main/kotlin/comments/CommentExtractorPSI.kt‎

Lines changed: 112 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,112 @@
1+
package com.github.codeql.comments
2+
3+
import com.github.codeql.*
4+
import com.github.codeql.utils.IrVisitorLookup
5+
import com.github.codeql.utils.Psi2IrFacade
6+
import com.github.codeql.utils.versions.getPsi2Ir
7+
import com.intellij.psi.PsiComment
8+
import com.intellij.psi.PsiElement
9+
import org.jetbrains.kotlin.config.KotlinCompilerVersion
10+
import org.jetbrains.kotlin.ir.IrElement
11+
import org.jetbrains.kotlin.ir.declarations.*
12+
import org.jetbrains.kotlin.kdoc.psi.api.KDoc
13+
import org.jetbrains.kotlin.lexer.KtTokens
14+
import org.jetbrains.kotlin.psi.KtVisitor
15+
import org.jetbrains.kotlin.psi.psiUtil.endOffset
16+
import org.jetbrains.kotlin.psi.psiUtil.startOffset
17+
18+
class CommentExtractorPSI(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) {
19+
// Returns true if it extracted the comments; false otherwise.
20+
fun extract(): Boolean {
21+
val psi2Ir = getPsi2Ir()
22+
if (psi2Ir == null) {
23+
logger.warn("Comments will not be extracted as Kotlin version is too old (${KotlinCompilerVersion.getVersion()})")
24+
return false
25+
}
26+
val ktFile = psi2Ir.getKtFile(file)
27+
if (ktFile == null) {
28+
return false
29+
}
30+
val commentVisitor = mkCommentVisitor(psi2Ir)
31+
ktFile.accept(commentVisitor)
32+
return true
33+
}
34+
35+
private fun mkCommentVisitor(psi2Ir: Psi2IrFacade): KtVisitor<Unit, Unit> =
36+
object : KtVisitor<Unit, Unit>() {
37+
override fun visitElement(element: PsiElement) {
38+
element.acceptChildren(this)
39+
40+
// Slightly hacky, but `visitComment` doesn't seem to visit comments with `tokenType` `KtTokens.DOC_COMMENT`
41+
if (element is PsiComment){
42+
visitCommentElement(element)
43+
}
44+
}
45+
46+
private fun visitCommentElement(comment: PsiComment) {
47+
val type: CommentType = when (comment.tokenType) {
48+
KtTokens.EOL_COMMENT -> {
49+
CommentType.SingleLine
50+
}
51+
KtTokens.BLOCK_COMMENT -> {
52+
CommentType.Block
53+
}
54+
KtTokens.DOC_COMMENT -> {
55+
CommentType.Doc
56+
}
57+
else -> {
58+
logger.warn("Unhandled comment token type: ${comment.tokenType}")
59+
return
60+
}
61+
}
62+
63+
val commentLabel = tw.getFreshIdLabel<DbKtcomment>()
64+
tw.writeKtComments(commentLabel, type.value, comment.text)
65+
val locId = tw.getLocation(comment.startOffset, comment.endOffset)
66+
tw.writeHasLocation(commentLabel, locId)
67+
68+
if (comment.tokenType != KtTokens.DOC_COMMENT) {
69+
return
70+
}
71+
72+
if (comment !is KDoc) {
73+
logger.warn("Unexpected comment type with DocComment token type.")
74+
return
75+
}
76+
77+
for (sec in comment.getAllSections()) {
78+
val commentSectionLabel = tw.getFreshIdLabel<DbKtcommentsection>()
79+
tw.writeKtCommentSections(commentSectionLabel, commentLabel, sec.getContent())
80+
val name = sec.name
81+
if (name != null) {
82+
tw.writeKtCommentSectionNames(commentSectionLabel, name)
83+
}
84+
val subjectName = sec.getSubjectName()
85+
if (subjectName != null) {
86+
tw.writeKtCommentSectionSubjectNames(commentSectionLabel, subjectName)
87+
}
88+
}
89+
90+
// Only storing the owner of doc comments:
91+
val ownerPsi = getKDocOwner(comment) ?: return
92+
93+
val owners = mutableListOf<IrElement>()
94+
file.accept(IrVisitorLookup(psi2Ir, ownerPsi, file), owners)
95+
96+
for (ownerIr in owners) {
97+
val ownerLabel = getLabel(ownerIr)
98+
if (ownerLabel != null) {
99+
tw.writeKtCommentOwners(commentLabel, ownerLabel)
100+
}
101+
}
102+
}
103+
104+
private fun getKDocOwner(comment: KDoc) : PsiElement? {
105+
val owner = comment.owner
106+
if (owner == null) {
107+
logger.warn("Couldn't get owner of KDoc. The comment is extracted without an owner.")
108+
}
109+
return owner
110+
}
111+
}
112+
}

‎java/kotlin-extractor/src/main/kotlin/utils/versions/v_1_4_32/CommentExtractorLighterAST.kt‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package com.github.codeql.comments
2+
3+
import com.github.codeql.*
4+
import org.jetbrains.kotlin.ir.declarations.*
5+
6+
class CommentExtractorLighterAST(fileExtractor: KotlinFileExtractor, file: IrFile, fileLabel: Label<out DbFile>): CommentExtractor(fileExtractor, file, fileLabel) {
7+
// We don't support LighterAST with old Kotlin versions
8+
fun extract(): Boolean {
9+
return false
10+
}
11+
}

0 commit comments

Comments
 (0)