123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177 |
- /*
- Copyright 2014 The Kubernetes Authors.
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
- http://www.apache.org/licenses/LICENSE-2.0
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- */
- package runtime
- import (
- "fmt"
- "net/http"
- "runtime"
- "sync"
- "time"
- "k8s.io/klog/v2"
- )
- var (
- // ReallyCrash controls the behavior of HandleCrash and defaults to
- // true. It's exposed so components can optionally set to false
- // to restore prior behavior. This flag is mostly used for tests to validate
- // crash conditions.
- ReallyCrash = true
- )
- // PanicHandlers is a list of functions which will be invoked when a panic happens.
- var PanicHandlers = []func(interface{}){logPanic}
- // HandleCrash simply catches a crash and logs an error. Meant to be called via
- // defer. Additional context-specific handlers can be provided, and will be
- // called in case of panic. HandleCrash actually crashes, after calling the
- // handlers and logging the panic message.
- //
- // E.g., you can provide one or more additional handlers for something like shutting down go routines gracefully.
- func HandleCrash(additionalHandlers ...func(interface{})) {
- if r := recover(); r != nil {
- for _, fn := range PanicHandlers {
- fn(r)
- }
- for _, fn := range additionalHandlers {
- fn(r)
- }
- if ReallyCrash {
- // Actually proceed to panic.
- panic(r)
- }
- }
- }
- // logPanic logs the caller tree when a panic occurs (except in the special case of http.ErrAbortHandler).
- func logPanic(r interface{}) {
- if r == http.ErrAbortHandler {
- // honor the http.ErrAbortHandler sentinel panic value:
- // ErrAbortHandler is a sentinel panic value to abort a handler.
- // While any panic from ServeHTTP aborts the response to the client,
- // panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log.
- return
- }
- // Same as stdlib http server code. Manually allocate stack trace buffer size
- // to prevent excessively large logs
- const size = 64 << 10
- stacktrace := make([]byte, size)
- stacktrace = stacktrace[:runtime.Stack(stacktrace, false)]
- if _, ok := r.(string); ok {
- klog.Errorf("Observed a panic: %s\n%s", r, stacktrace)
- } else {
- klog.Errorf("Observed a panic: %#v (%v)\n%s", r, r, stacktrace)
- }
- }
- // ErrorHandlers is a list of functions which will be invoked when a nonreturnable
- // error occurs.
- // TODO(lavalamp): for testability, this and the below HandleError function
- // should be packaged up into a testable and reusable object.
- var ErrorHandlers = []func(error){
- logError,
- (&rudimentaryErrorBackoff{
- lastErrorTime: time.Now(),
- // 1ms was the number folks were able to stomach as a global rate limit.
- // If you need to log errors more than 1000 times a second you
- // should probably consider fixing your code instead. :)
- minPeriod: time.Millisecond,
- }).OnError,
- }
- // HandlerError is a method to invoke when a non-user facing piece of code cannot
- // return an error and needs to indicate it has been ignored. Invoking this method
- // is preferable to logging the error - the default behavior is to log but the
- // errors may be sent to a remote server for analysis.
- func HandleError(err error) {
- // this is sometimes called with a nil error. We probably shouldn't fail and should do nothing instead
- if err == nil {
- return
- }
- for _, fn := range ErrorHandlers {
- fn(err)
- }
- }
- // logError prints an error with the call stack of the location it was reported
- func logError(err error) {
- klog.ErrorDepth(2, err)
- }
- type rudimentaryErrorBackoff struct {
- minPeriod time.Duration // immutable
- // TODO(lavalamp): use the clock for testability. Need to move that
- // package for that to be accessible here.
- lastErrorTimeLock sync.Mutex
- lastErrorTime time.Time
- }
- // OnError will block if it is called more often than the embedded period time.
- // This will prevent overly tight hot error loops.
- func (r *rudimentaryErrorBackoff) OnError(error) {
- now := time.Now() // start the timer before acquiring the lock
- r.lastErrorTimeLock.Lock()
- d := now.Sub(r.lastErrorTime)
- r.lastErrorTime = time.Now()
- r.lastErrorTimeLock.Unlock()
- // Do not sleep with the lock held because that causes all callers of HandleError to block.
- // We only want the current goroutine to block.
- // A negative or zero duration causes time.Sleep to return immediately.
- // If the time moves backwards for any reason, do nothing.
- time.Sleep(r.minPeriod - d)
- }
- // GetCaller returns the caller of the function that calls it.
- func GetCaller() string {
- var pc [1]uintptr
- runtime.Callers(3, pc[:])
- f := runtime.FuncForPC(pc[0])
- if f == nil {
- return "Unable to find caller"
- }
- return f.Name()
- }
- // RecoverFromPanic replaces the specified error with an error containing the
- // original error, and the call tree when a panic occurs. This enables error
- // handlers to handle errors and panics the same way.
- func RecoverFromPanic(err *error) {
- if r := recover(); r != nil {
- // Same as stdlib http server code. Manually allocate stack trace buffer size
- // to prevent excessively large logs
- const size = 64 << 10
- stacktrace := make([]byte, size)
- stacktrace = stacktrace[:runtime.Stack(stacktrace, false)]
- *err = fmt.Errorf(
- "recovered from panic %q. (err=%v) Call stack:\n%s",
- r,
- *err,
- stacktrace)
- }
- }
- // Must panics on non-nil errors. Useful to handling programmer level errors.
- func Must(err error) {
- if err != nil {
- panic(err)
- }
- }
|