Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package org.apache.spark.deploy.history

import java.io.IOException
import java.net.URI
import java.nio.charset.CharacterCodingException
import java.util.ServiceLoader

import scala.jdk.CollectionConverters._
Expand Down Expand Up @@ -140,7 +141,13 @@ class EventLogFileCompactor(
// check happens between files, so that file itself is still read to the end.
val replayedAll = files.forall { log =>
val completed = Utils.tryWithResource(EventLogFileReader.openEventLog(log, fs)) { in =>
bus.replay(in, log.getName)
try {
bus.replay(in, log.getName)
} catch {
case e: CharacterCodingException =>
logWarning(log"Event log file ${MDC(LogKeys.PATH, log)} contains malformed input")
throw e
}
}
if (bus.numSkippedLines > 0) {
logWarning(log"Skipping compaction of event log files in " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package org.apache.spark.deploy.history

import java.io.{ByteArrayInputStream, File, FileNotFoundException, InputStream, IOException, SequenceInputStream}
import java.lang.{Long => JLong}
import java.nio.charset.CharacterCodingException
import java.util.{Date, NoSuchElementException, ServiceLoader}
import java.util.concurrent.{ConcurrentHashMap, ExecutorService, TimeUnit}
import java.util.zip.ZipOutputStream
Expand Down Expand Up @@ -946,7 +947,18 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
if (info.lastEvaluatedForCompaction.isEmpty ||
info.lastEvaluatedForCompaction.get < lastIndex) {
// haven't tried compaction for this index, do compaction
fileCompactor.compact(reader.listEventLogFiles)
try {
fileCompactor.compact(reader.listEventLogFiles)
} catch {
case e: CharacterCodingException =>
// rewrite() only runs after initializeBuilders has replayed every file to compact
// without error, so no files have been written or deleted at this point.
// Only rolled event log files are compacted and they are no longer modified,
// so retrying would fail the same way. Record this index as evaluated to
// avoid replaying the same files on every scan.
logWarning(log"Skipping compaction for ${MDC(PATH, rootPath)} as the event " +
log"log files contain malformed input", e)
}
listing.write(info.copy(lastEvaluatedForCompaction = Some(lastIndex)))
}
} catch {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ package org.apache.spark.deploy.history

import java.io._
import java.nio.charset.StandardCharsets
import java.nio.file.Files
import java.nio.file.{Files, StandardOpenOption}
import java.util.{Date, Locale}
import java.util.concurrent.TimeUnit
import java.util.zip.{ZipInputStream, ZipOutputStream}
Expand All @@ -30,6 +30,7 @@ import org.apache.hadoop.fs.{FileStatus, FileSystem, FSDataInputStream, Path}
import org.apache.hadoop.hdfs.{DFSInputStream, DistributedFileSystem}
import org.apache.hadoop.ipc.{CallerContext => HadoopCallerContext}
import org.apache.hadoop.security.AccessControlException
import org.apache.logging.log4j.Level
import org.mockito.ArgumentMatchers.{any, argThat}
import org.mockito.Mockito.{doThrow, mock, spy, verify, when}
import org.scalatest.PrivateMethodTester
Expand All @@ -40,7 +41,7 @@ import org.scalatest.matchers.should.Matchers._
import org.apache.spark.{JobExecutionStatus, SecurityManager, SPARK_VERSION, SparkConf, SparkFunSuite}
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.deploy.history.EventLogTestHelper._
import org.apache.spark.internal.config.DRIVER_LOG_DFS_DIR
import org.apache.spark.internal.config.{DRIVER_LOG_DFS_DIR, EVENT_LOG_COMPRESS}
import org.apache.spark.internal.config.History._
import org.apache.spark.internal.config.UI.{ADMIN_ACLS, ADMIN_ACLS_GROUPS, UI_VIEW_ACLS, UI_VIEW_ACLS_GROUPS, USER_GROUPS_MAPPING}
import org.apache.spark.io._
Expand Down Expand Up @@ -1685,6 +1686,66 @@ abstract class FsHistoryProviderSuite extends SparkFunSuite with Matchers with P
}
}

test("SPARK-59747: don't retry compaction of event log files with malformed input") {
withTempDir { dir =>
val conf = createTestConf().set(FAST_IN_PROGRESS_PARSING, true)
conf.set(HISTORY_LOG_DIR, dir.getAbsolutePath)
conf.set(EVENT_LOG_ROLLING_MAX_FILES_TO_RETAIN, 1)
conf.set(EVENT_LOG_COMPACTION_SCORE_THRESHOLD, 0.0d)
// Keep the event log files uncompressed so that the bytes appended below are decoded as-is.
conf.set(EVENT_LOG_COMPRESS, false)
val hadoopConf = SparkHadoopUtil.newConfiguration(conf)
val fs = new Path(dir.getAbsolutePath).getFileSystem(hadoopConf)

val provider = new FsHistoryProvider(conf)

val writer = new RollingEventLogFilesWriter("app", None, dir.toURI, conf, hadoopConf)
writer.start()

// Keep the writer running so fast listing replay halts after the environment update.
// Pad the first file so the malformed bytes are not decoded by listing's read-ahead,
// but are decoded when compaction replays the file in full.
writeEventsToRollingWriter(writer, Seq(
SparkListenerApplicationStart("app", Some("app"), 0, "user", None),
SparkListenerEnvironmentUpdate(Map(
"Spark Properties" -> Seq.empty,
"Hadoop Properties" -> Seq.empty,
"JVM Information" -> Seq.empty,
"System Properties" -> Seq.empty,
"Metrics Properties" -> Seq.empty,
"Classpath Entries" -> Seq.empty))) ++
(1 to 1000).map(SparkListenerJobStart(_, 0, Seq.empty)), rollFile = true)

val firstFile = EventLogFileReader(fs, new Path(writer.logPath)).get.listEventLogFiles.head
Files.write(new File(firstFile.getPath.toUri).toPath,
Array[Byte](0xC3.toByte, 0x28.toByte, '\n'.toByte), StandardOpenOption.APPEND)

writeEventsToRollingWriter(writer, Seq(SparkListenerJobEnd(1, 1, JobSucceeded)),
rollFile = false)

val logAppender = new LogAppender("malformed event log compaction")
withLogAppender(logAppender, loggerNames = Seq(classOf[EventLogFileCompactor].getName)) {
updateAndCheck(provider) { list =>
assert(list.map(_.id) === Seq("app"))
val logFiles = EventLogFileReader(fs, new Path(writer.logPath)).get.listEventLogFiles
assert(!logFiles.exists(file => EventLogFileWriter.isCompacted(file.getPath)))
assert(logFiles.map { file =>
RollingEventLogFilesWriter.getEventLogFileIndex(file.getPath.getName)
} === Seq(1, 2))
val info = provider.listing.read(classOf[LogInfo], writer.logPath)
assert(info.lastEvaluatedForCompaction === Some(2))
}
}
assert(logAppender.loggingEvents.exists { event =>
event.getLevel == Level.WARN && event.getMessage.getFormattedMessage ==
s"Event log file ${firstFile.getPath} contains malformed input"
})

writer.stop()
provider.stop()
}
}

test("SPARK-33146: don't let one bad rolling log folder prevent loading other applications") {
withTempDir { dir =>
val conf = createTestConf(true)
Expand Down