Coverage

93%
378
353
25

src/clone.coffee

100%
18
18
0
LineHitsSource
1
21shjs = require 'shelljs'
31Blamer = require 'blamer'
41Promise = require 'bluebird'
5
61class Clone
7 constructor: (
8 @firstFile,
9 @secondFile,
10 @firstFileStart,
11 @secondFileStart,
12 @linesCount,
13 @tokensCount)->
1460 @firstFileAnnotatedCode = {}
1560 @secondFileAnnotatedCode = {}
16
17
18 getLines: (isFirstFile = yes) ->
19120 code = shjs.cat(if isFirstFile then @firstFile else @secondFile)
20120 start = if isFirstFile then @firstFileStart else @secondFileStart
21120 lines = code.split '\n'
22120 end = start + @linesCount
23120 lines[start..end].join("\n")
24
25 blame: ->
2630 blamer = new Blamer
2730 Promise.all([
28 blamer.blameByFile(@firstFile)
29 blamer.blameByFile(@secondFile)
30 ]).then (results) =>
3130 @firstFileAnnotatedCode[line] = annotation for line, annotation of results[0][@firstFile] when @lineInRange(line, @firstFileStart)
3230 @secondFileAnnotatedCode[line] = annotation for line, annotation of results[1][@secondFile] when @lineInRange(line, @secondFileStart)
3330 return @
34
355255 lineInRange: (line, fileStart) -> 0 + line >= fileStart and 0 + line <= fileStart + @linesCount
36
371exports.Clone = Clone
38

src/detector.coffee

100%
6
6
0
LineHitsSource
1
21Map = require('./map').Map
3
41class Detector
5
6 constructor: (@strategy) ->
7
8 start: (files = [], minLines = 5, minTokens = 70)->
947 map = new Map()
1047 @strategy.detect map, file, minLines, minTokens for file in files
1147 map
12
131exports.Detector = Detector
14

src/jscpd.coffee

100%
36
36
0
LineHitsSource
11logger = require 'winston'
2
31{Detector} = require './detector'
41{Strategy} = require './strategy'
51{Report} = require './report'
6
71optionsPreprocessor = require './preprocessors/options'
81filesPreprocessor = require './preprocessors/files'
91debugPreprocessor = require './preprocessors/debug'
10
111Promise = require "bluebird"
12
131class jscpd
14
15 preProcessors: [
16 optionsPreprocessor
17 filesPreprocessor
18 debugPreprocessor
19 ]
20
21 LANGUAGES: []
22
23 execPreProcessors: (list) ->
2449 logger.profile 'Preprocessors running time:'
2549 preProcessor @ for preProcessor in list
2649 logger.profile 'Preprocessors running time:'
27
28 run: (options) ->
2948 @options = options
3048 @execPreProcessors @preProcessors
31
3248 unless @options.debug
3347 logger.profile 'Scanning for duplicates time:'
3447 logger.info "Scanning #{@options.selectedFiles.length} files for duplicates..." if @options.selectedFiles.length
35
3647 strategy = new Strategy @options
3747 detector = new Detector strategy
38
3947 report = new Report @options
40
4147 codeMap = detector.start @options.selectedFiles, @options['min-lines'], @options['min-tokens']
42
4347 logger.profile 'Scanning for duplicates time:'
4447 logger.info 'Scanning... done!\n'
45
4647 logger.profile 'Generate report time:'
4747 logger.info 'Start report generation...\n'
48
4947 generateReport = =>
5047 reportResult = report.generate codeMap
5147 logger.profile 'Generate report time:'
5247 @report = reportResult
5347 @map = codeMap
5447 report: @report, map: @map
55
5647 if @options.blame
5716 Promise
5830 .all(clone.blame() for clone in codeMap.clones)
59 .then generateReport
60 .error generateReport
61 else
6231 return generateReport()
63
641module.exports = jscpd
65

src/map.coffee

100%
21
21
0
LineHitsSource
1
21class Map
3 constructor: ->
447 @clones = []
547 @clonesByFile = {}
647 @numberOfDuplication = 0
747 @numberOfLines = 0
847 @numberOfFiles = 0
9
10 addClone: (clone)->
1160 @clones.push clone
1260 @numberOfDuplication = @numberOfDuplication + clone.linesCount
13
1460 if clone.firstFile of @clonesByFile
1526 @clonesByFile[clone.firstFile].push clone.firstFile
16 else
1734 @clonesByFile[clone.firstFile] = [clone.firstFile]
1834 @numberOfFiles++
19
2060 if clone.secondFile of @clonesByFile
2120 @clonesByFile[clone.secondFile].push clone
22 else
2340 @clonesByFile[clone.secondFile] = [clone]
2440 @numberOfFiles++
25
26 getPercentage: ->
2760 result = 0
2860 if @numberOfLines > 0
2930 result = @numberOfDuplication / @numberOfLines * 100
3060 result.toFixed 2
31
32
331exports.Map = Map
34

src/preprocessors/debug.coffee

100%
12
12
0
LineHitsSource
12logger = require "winston"
2
3
42debugPreprocessor = (jscpd) ->
549 if jscpd.options.debug
62 logger.info '----------------------------------------'
72 logger.info 'Options:'
82 logger.info "#{name} = #{option}" for name, option of jscpd.options when name isnt 'selectedFiles'
92 logger.info '----------------------------------------'
102 logger.info 'Files:'
112 logger.info file for file in jscpd.options.selectedFiles
122 logger.info '----------------------------------------'
132 logger.info 'Run without debug option for start detection process'
14
15
162module.exports = debugPreprocessor
17
18

src/preprocessors/files.coffee

100%
27
27
0
LineHitsSource
15_ = require "underscore"
25path = require "path"
35glob = require "glob"
4
55findFiles = (jscpd) ->
652 files = []
752 excluded_files = []
8
952 for pattern in jscpd.options.patterns
1052 files = _.union files, glob.sync(pattern, cwd: jscpd.options.path)
11
1252 if jscpd.options.exclude and jscpd.options.exclude.length > 0
1322 for pattern in jscpd.options.exclude
1440 excluded_files = _.union excluded_files, glob.sync(pattern, cwd: jscpd.options.path)
15
1652 files = _.difference files, excluded_files
1752 files = _.map files, (file) -> path.normalize "#{jscpd.options.path}/#{file}"
1852 return files
19
205prepareOptions = (jscpd) ->
2152 if jscpd.options.files is null
222 jscpd.options.patterns = ["**/*.+(#{jscpd.options.extensions.join '|'})"]
23 else
2450 unless Array.isArray(jscpd.options.files)
2531 jscpd.options.patterns = [jscpd.options.files]
26 else
2719 jscpd.options.patterns = jscpd.options.files
2852 if jscpd.options.exclude isnt null
2952 unless Array.isArray(jscpd.options.exclude)
304 jscpd.options.exclude = [jscpd.options.exclude]
31
32
335filesPreprocessor = (jscpd) ->
3452 prepareOptions jscpd
3552 jscpd.options.selectedFiles = findFiles jscpd
36
375module.exports = filesPreprocessor
38
39

src/preprocessors/options.coffee

100%
28
28
0
LineHitsSource
14_ = require "underscore"
24yaml = require 'js-yaml'
34fs = require 'fs'
44path = require 'path'
5
64TokenizerFactory = require '../tokenizer/TokenizerFactory'
7
84prepareOptions = (options, config) ->
951 optionsNew = _.extend optionsPreprocessor.default, config
10
1151 for key, value of options
12297 if not (value is null)
13297 optionsNew[key] = value
14
1551 if typeof optionsNew.languages is 'string'
161 optionsNew.languages = optionsNew.languages.split ','
17
1851 optionsNew.extensions = TokenizerFactory::getExtensionsByLanguages(optionsNew.languages)
1951 return optionsNew
20
214readConfig = (file) ->
2252 file = path.normalize file
2352 try
2452 doc = yaml.safeLoad fs.readFileSync(file, 'utf8')
2550 doc.config_file = file
2650 return doc
27 catch error
282 return false
29
30
314optionsPreprocessor = (jscpd) ->
3251 config = readConfig('.cpd.yaml') or readConfig('.cpd.yml') or {}
3351 options = prepareOptions jscpd.options, config
3451 options.path = options.path or process.cwd()
3551 jscpd.options = options
36
374optionsPreprocessor.default =
38 languages: Object.keys TokenizerFactory::LANGUAGES
39 verbose: off
40 debug: off
41 files: null
42 exclude: null
43
444module.exports = optionsPreprocessor
45
46

src/report.coffee

88%
26
23
3
LineHitsSource
11fs = require 'fs'
21logger = require 'winston'
31path = require 'path'
4
5
6
71class Report
8
9 constructor: (@options) ->
10
1147 reporter = @options.reporter
12
1347 ext = @options.output.split('.').pop() if @options.output
14
1547 if ext is 'xml' and reporter is 'json' or
16 ext is 'json' and reporter is 'xml'
17
180 logger.warn "output file extention '#{@options.output}'
190 does not match reporter '#{reporter}'"
20
2147 switch reporter
2230 when 'xml' then reporter = './reporters/xml-pmd'
2315 when 'json' then reporter = './reporters/json'
24 else
252 cwd = process.cwd()
262 reporter = path.normalize reporter
272 isAbsolute = reporter.indexOf(cwd) is 0
282 reporter = path.join(cwd, reporter) unless isAbsolute
29
3047 @reporter = require reporter
3147 @stdReporter = require './reporters/_std-log'
32
33 generate: (@map) ->
3447 [raw, dump, log] = @reporter @options
3547 log = @stdReporter() unless log
36
3747 logger.info log
3847 if @options.output
390 fs.writeFileSync(@options.output, dump or raw)
40 else
4147 logger.warn 'output file is not provided'
42
4347 return raw or dump
44
451exports.Report = Report
46

src/reporters/_std-log.coffee

61%
44
27
17
LineHitsSource
11require 'colors'
21Table = require 'cli-table'
3
41TABLE_CONFIGURATION = chars:
5 'mid': ''
6 'left-mid': ''
7 'mid-mid': ''
8 'right-mid': ''
9 'top': ''
10 'top-mid': ''
11 'top-left': ''
12 'top-right': ''
13 'bottom': ''
14 'bottom-mid': ''
15 'bottom-left': ''
16 'bottom-right': ''
17 'left': ''
18 'right': ''
191compareDates = (firstDate, secondDate)->
200 firstDate = new Date firstDate
210 secondDate = new Date secondDate
220 switch yes
230 when firstDate < secondDate then "=>"
240 when firstDate > secondDate then "<="
250 else "=="
26
271module.exports = ->
28
2945 clog = ''
3045 verbose = @options.verbose
31
3245 for clone in @map.clones
3360 do (clone) ->
3460 table = ''
3560 firstFile = clone.firstFile
3660 secondFile = clone.secondFile
37
3860 if verbose
390 table = new Table TABLE_CONFIGURATION
40
410 fragment = clone.getLines().split("\n").reduce (tbl, current, lineNumber) ->
420 firstFileLine = clone.firstFileStart + lineNumber
430 secondFileLine = clone.secondFileStart + lineNumber
440 if Object.keys(clone.firstFileAnnotatedCode) > 0
450 tbl.push [
46 firstFileLine
47 clone.firstFileAnnotatedCode[firstFileLine].author
48 compareDates clone.firstFileAnnotatedCode[firstFileLine].date, clone.secondFileAnnotatedCode[secondFileLine].date
49 secondFileLine
50 clone.secondFileAnnotatedCode[secondFileLine].author
51 current.dim
52 ]
53 else
540 tbl.push [
55 firstFileLine
56 secondFileLine
57 current.dim
58 ]
590 tbl
60 , table
61
6260 clog = "#{clog}\n\t-
6360 #{firstFile.green.bold}:
6460 #{clone.firstFileStart}-#{clone.firstFileStart + clone.linesCount}\n\t
6560 #{secondFile.green.bold}:
6660 #{clone.secondFileStart}-#{clone.secondFileStart + clone.linesCount}\n"
67
6860 clog = "#{clog}\n#{fragment.toString()}\n" if verbose
69
7045 percent = @map.getPercentage()
71
7245 log = "Found #{@map.clones.length} exact clones with
7345 #{@map.numberOfDuplication} duplicated lines in
7445 #{@map.numberOfFiles} files\n #{clog}\n\n
7545 #{percent}% (#{@map.numberOfDuplication} lines)
76 duplicated lines out of
7745 #{@map.numberOfLines} total lines of code.\n"
78
7945 if @options.limit <= percent
800 console.error log
810 console.error "ERROR: jscpd found too many duplicates over threshold"
820 process.exit(1)
83
8445 return log
85

src/reporters/json.coffee

75%
8
6
2
LineHitsSource
11_ = require 'underscore'
2
3# @map {Object} report object
4# @options {Object} Report class options
51module.exports = ->
6
715 duplicates = []
8
915 for clone in @map.clones
100 do (clone) ->
11
120 duplicates.push
13
14 lines: clone.linesCount
15 tokens: clone.tokensCount
16 firstFile:
17 start: clone.firstFileStart
18 name: clone.firstFile
19 secondFile:
20 start: clone.secondFileStart
21 name: clone.secondFile
22 fragment: _.escape(clone.getLines())
23
2415 json =
25 duplicates: duplicates
26 statistics:
27 clones: @map.clones.length
28 duplications: @map.numberOfDuplication
29 files: @map.numberOfFiles
30 percentage: @map.getPercentage()
31 lines: @map.numberOfLines
32
3315 [
34 json
35 JSON.stringify(json)
36 ]
37

src/reporters/xml-pmd.coffee

100%
19
19
0
LineHitsSource
11_ = require 'underscore'
21Promise = require 'bluebird'
3
4# @map {Object} report object
5# @options {Object} Report class options
61module.exports = (options) ->
730 xmlDoc = "<?xml version='1.0' encoding='UTF-8' ?>"
830 if options['xsl-href'] then xmlDoc += '<?xml-stylesheet type="text/xsl" href="' + options['xsl-href'] + '"?>'
930 xmlDoc += '<pmd-cpd>'
10
11# Promise.all(@map.clones.map (clone) -> clone.blame()).then (clones) ->
12# console.log clones
13
1430 for clone in @map.clones
1560 do (clone) ->
1660 firstFragment = _.escape clone.getLines(yes)
1760 secondFragment = _.escape clone.getLines(no)
1860 xmlDoc = "#{xmlDoc}
1960 <duplication lines='#{clone.linesCount}' tokens='#{clone.tokensCount}'>
2060 <file path='#{clone.firstFile}' line='#{clone.firstFileStart}'>#{clone.firstFileAnnotatedCode}
2160 <codefragment><![CDATA[#{firstFragment}]]></codefragment>
22 </file>
2360 <file path='#{clone.secondFile}' line='#{clone.secondFileStart}'>#{clone.secondFileAnnotatedCode}
2460 <codefragment><![CDATA[#{secondFragment}]]></codefragment>
25 </file>
2660 <codefragment><![CDATA[#{firstFragment}]]></codefragment>
27 </duplication>"
2830 xmlDoc = xmlDoc + '</pmd-cpd>'
29
3030 [xmlDoc]
31

src/storage/StorageMemory.coffee

100%
7
7
0
LineHitsSource
1
21class StorageMemory
3
4 constructor: ->
547 @codeHashes = {}
6
7 addHash: (hash, file, line, language)->
811030 @codeHashes[language] = @codeHashes[language] ? {}
911030 @codeHashes[language][hash] = line: line, file: file
10
11 hasHash: (hash, language) ->
1220528 @codeHashes[language] and hash of @codeHashes[language]
13
14 getHash: (hash, language) ->
1560 @codeHashes[language]?[hash]
16
17
181module.exports = StorageMemory

src/strategy.coffee

98%
50
49
1
LineHitsSource
11shjs = require 'shelljs'
21TokenizerFactory = require './tokenizer/TokenizerFactory'
31crypto = require 'crypto'
4
51Storage = require './storage/StorageMemory'
6
71Clone = require('./clone').Clone
8
91class Strategy
10
11 constructor: (options) ->
1247 @options = options
1347 @languages = options.languages
1447 @storage = new Storage()
15
16 detect: (map, file, @minLines, @minTokens) ->
171170 tokenizer = TokenizerFactory::makeTokenizer file, @languages
181170 unless tokenizer
191092 return no
2078 language = tokenizer.getType()
21
2278 if shjs.test '-f', file
2378 code = shjs.cat file
24 else
250 return no
26
2778 lines = code.split '\n'
2878 map.numberOfLines = map.numberOfLines + lines.length
29
3078 tokenizer.skipComments = @options['skip-comments']
31
3278 {tokensPositions, currentMap} = tokenizer.tokenize(code).generateMap()
33
3478 firstLine = 0
3578 tokenNumber = 0
3678 isClone = false
37
3878 while tokenNumber <= tokensPositions.length - @minTokens
3920528 mapFrame = currentMap.substring tokenNumber * 33, tokenNumber * 33 + @minTokens * 33
4020528 hash = crypto.createHash('md5').update(mapFrame).digest('hex')
41
4220528 if @storage.hasHash hash, language
439498 isClone = true
449498 if firstLine is 0
4560 firstLine = tokensPositions[tokenNumber]
4660 firstHash = hash
4760 firstToken = tokenNumber
48 else
4911030 if isClone
5028 lastToken = tokenNumber + @minTokens - 2
5128 @addClone(
52 map,
53 file,
54 firstHash,
55 firstToken,
56 lastToken,
57 firstLine,
58 tokensPositions[lastToken],
59 language
60 )
6128 firstLine = 0
6228 isClone = false
6311030 @storage.addHash hash, file, tokensPositions[tokenNumber], language
6420528 tokenNumber = tokenNumber + 1
65
6678 if isClone
6732 lastToken = tokenNumber + @minTokens - 2
6832 @addClone(
69 map,
70 file,
71 firstHash,
72 firstToken,
73 lastToken,
74 firstLine,
75 tokensPositions[lastToken],
76 language
77 )
7832 isClone = false
79
80 addClone: (map, file, hash, firstToken, lastToken, firstLine, lastLine, language) ->
8160 hashInfo = @storage.getHash(hash, language)
8260 fileA = hashInfo.file
8360 firstLineA = hashInfo.line
8460 numLines = lastLine + 1 - firstLine
8560 if numLines >= @minLines and (fileA isnt file or firstLineA isnt firstLine)
8660 map.addClone new Clone(
87 fileA,
88 file,
89 firstLineA,
90 firstLine,
91 numLines,
92 lastToken - firstToken + 1
93 )
94
95
961exports.Strategy = Strategy
97

src/tokenizer/TokenizerBase.coffee

100%
14
14
0
LineHitsSource
1
21crypto = require 'crypto'
3
41class TokenizerBase
5
6 constructor: ->
715 @skipComments = no
815 @tokenTypes = []
9
10 tokenize: (code) ->
11
12 getType: () ->
13
14 generateMap: () ->
15
1635804 isEmptyToken: (value) -> value.replace(/^\s+|\s+$/g, '').length is 0
17
18 validToken: (type) ->
1935804 (@type in ['coffeescript', 'python', 'ruby'] or type isnt 'empty') and (not @skipComments or type isnt 'comment')
20
21 getTokenTypeId: (name) ->
2225826 result = 0
2325826 if name in @tokenTypes
2425689 result = @tokenTypes.indexOf(name)
25 else
26137 result = @tokenTypes.length
27137 @tokenTypes.push name
2825826 result
29
30 createMap: (type, value) ->
3125826 String.fromCharCode(@getTokenTypeId(type)) +
32 crypto
33 .createHash('md5')
34 .update(value.toString())
35 .digest('hex')
36
37
381module.exports = TokenizerBase
39

src/tokenizer/TokenizerCodeMirror.coffee

97%
38
37
1
LineHitsSource
11vm = require "vm"
21fs = require "fs"
3
41TokenizerBase = require './TokenizerBase'
51logger = require 'winston'
61CodeMirror = require("codemirror/addon/runmode/runmode.node.js")
7
81CodeMirror.loadMode = (name) ->
980 filename = require.resolve("codemirror/mode/" + name + "/" + name + ".js")
1072 modeDef = null
1172 try
1272 modeDef = fs.readFileSync(filename, "utf8")
13 catch err
140 throw new Error(name + " mode isn't shipped with CodeMirror")
1572 vm.runInNewContext modeDef,
16 CodeMirror: CodeMirror
17
181CodeMirror.loadMode 'xml'
191CodeMirror.loadMode 'clike'
20
21
221class TokenizerCodeMirror extends TokenizerBase
231 @type = null
24
25 setType: (type) ->
2615 @type = type
27
28 loadType: (type) ->
2978 try
3078 CodeMirror.loadMode type
31 catch e
328 if e.code is 'MODULE_NOT_FOUND'
338 logger.debug "#{e}"
3478 @
35
36 tokenize: (code) =>
3778 @tokens = []
38
3978 @loadType @type
40
4178 CodeMirror.runMode code, @type, (value, tokenType, lineNumber) =>
4241690 return if not lineNumber
4335804 tokenType = if @isEmptyToken value then 'empty' else tokenType
4435804 tokenType = tokenType ? 'default'
4535804 @tokens.push [tokenType, value, lineNumber]
4678 @
47
4878 getType: -> @type
49
50 generateMap: ->
5178 currentMap = ""
5278 tokensPositions = []
5378 for [type, value, lineNumber] in @tokens when @validToken type
5425826 tokensPositions.push lineNumber
5525826 currentMap = currentMap + @createMap type, value
56
5778 tokensPositions: tokensPositions, currentMap: currentMap
58
59
601module.exports = TokenizerCodeMirror
61

src/tokenizer/TokenizerFactory.coffee

95%
24
23
1
LineHitsSource
1
21TokenizerCodeMirror = require './TokenizerCodeMirror'
3
41class TokenizerFactory
5
6 tokenizers: {}
7
8 LANGUAGES:
9 javascript: ['js', 'es', 'es6']
10 typescript: ['ts']
11 jsx: ['jsx']
12 haxe: ['hx', 'hxml']
13 coffeescript: ['coffee']
14 ruby: ['rb']
15 php: ['php', 'phtml']
16 python: ['py']
17 css: ['less', 'css', 'scss']
18 java: ['java']
19 csharp: ['cs']
20 go: ['go']
21 clike: ['cpp', 'c']
22 htmlmixed: ['html', 'htm']
23 yaml: ['yaml', 'yml']
24
25
26 getLanguageByExtension: (extension) ->
271181 for language of TokenizerFactory::LANGUAGES
289159 return language if extension in TokenizerFactory::LANGUAGES[language]
290 return no
30
31 getExtensionsByLanguages: (languages) ->
3251 languages = [languages] if typeof languages is 'string'
3351 result = []
3451 result.push TokenizerFactory::LANGUAGES[language]... for language of TokenizerFactory::LANGUAGES when language in languages
3551 return result
36
37 makeTokenizer: (filename, supportedLanguages) ->
381181 extension = ''
391181 matches = filename.match /\.(\w*)$/
401181 extension = matches[1]?.toLowerCase() if matches
41
421181 language = TokenizerFactory::getLanguageByExtension extension
43
441181 return off if language not in supportedLanguages
45
4688 if language not of TokenizerFactory::tokenizers
4715 switch language
48 when "csharp", "java", "csrc"
492 TokenizerFactory::tokenizers[language] = new TokenizerCodeMirror()
502 TokenizerFactory::tokenizers[language].setType "text/x-#{language}"
51 when "typescript", 'jsx'
522 TokenizerFactory::tokenizers[language] = new TokenizerCodeMirror()
532 TokenizerFactory::tokenizers[language].setType "javascript"
54 else
5511 TokenizerFactory::tokenizers[language] = new TokenizerCodeMirror()
5611 TokenizerFactory::tokenizers[language].setType language
57
5888 TokenizerFactory::tokenizers[language]
59
601module.exports = TokenizerFactory
61